fsmonitor.py (26597B)
1 #!/usr/bin/python 2 3 # a small program to test the possibilities to monitor the file system and 4 # log changes on Windowsm Linux, and OSX 5 # 6 # Originally written by Christoph Gohle (2010) 7 # Modified by Gene Horodecki for Windows 8 # Further modified by Benjamin Pierce 9 # should be distributed under GPL 10 11 import sys 12 import os 13 import stat 14 import threading 15 from optparse import OptionParser 16 from time import time, sleep 17 18 def mydebug(fmt, *args, **kwds): 19 if not op.debug: 20 return 21 22 if args: 23 fmt = fmt % args 24 25 elif kwds: 26 fmt = fmt % kwds 27 28 print >>sys.stderr, fmt 29 30 def mymesg(fmt, *args, **kwds): 31 if not op.verbose: 32 return 33 34 if args: 35 fmt = fmt % args 36 37 elif kwds: 38 fmt = fmt % kwds 39 40 print >>sys.stdout, fmt 41 42 def timer_callback(timer, streamRef): 43 mydebug("CFAbsoluteTimeGetCurrent() => %.3f", CFAbsoluteTimeGetCurrent()) 44 mydebug("FSEventStreamFlushAsync(streamRef = %s)", streamRef) 45 FSEventStreamFlushAsync(streamRef) 46 47 def update_changes(result): 48 mydebug('Update_changes: absresult = %s',result) 49 #print('absresult',result) 50 result = [mangle_filename(path) for path in result] 51 mydebug('Update_changes: mangled = %s',result) 52 #print('magnled', result) 53 result = [relpath(op.root,path) for path in result] 54 #print('relative to root',result) 55 mydebug('Update_changes: relative to root = %s',result) 56 57 try: 58 f = open(op.absoutfile,'a') 59 for path in result: 60 f.write(path+'\n') 61 f.close() 62 except IOError: 63 mymesg('failed to open log file %s for writing',op.outfile) 64 65 def update_changes_nomangle(result): 66 # In win32 there are no symlinks, therefore file mangling 67 # is not required 68 69 # remove root from the path: 70 result = relpath(op.root,result) 71 72 mydebug('Changed paths: %s\n',result) 73 try: 74 # Windows hack: open in binary mode 75 f = open(op.absoutfile,'ab') 76 f.write(result+'\n') 77 f.close() 78 except IOError: 79 mymesg('failed to open log file %s for writing',op.outfile) 80 81 def mangle_filename(path): 82 """because the FSEvents system returns 'real' paths we have to figure out 83 if they have been aliased by a symlink and a 'follow' directive in the unison 84 configuration or from the command line. 85 This is done here for path. The return value is the path name using symlinks 86 """ 87 try: 88 op.symlinks 89 except AttributeError: 90 make_symlinks() 91 #now lets do it 92 result = path 93 for key in op.symlinks: 94 #print path, key 95 if path.startswith(key): 96 result = os.path.join(op.root,os.path.join(op.symlinks[key]+path[len(key):])) 97 #print 'Match!', result 98 99 return result 100 101 def make_symlinks(): 102 #lets create a dictionary of symlinks that are treated transparently here 103 op.symlinks = {} 104 fl = op.follow 105 try: 106 foll = [f.split(' ',1) for f in fl] 107 except TypeError: 108 foll = [] 109 for k,v in foll: 110 if not k=='Path': 111 mymesg('We don\'t support anything but path specifications in follow directives. Especially not %s',k) 112 else: 113 p = v.strip('{}') 114 if not p[-1]=='/': 115 p+='/' 116 op.symlinks[os.path.realpath(os.path.join(op.root,p))]=p 117 mydebug('make_symlinks: symlinks to follow %s',op.symlinks) 118 119 120 def relpath(root,path): 121 """returns the path relative to root (which should be absolute) 122 if it is not a path below root or if root is not absolute it returns None 123 """ 124 125 if not os.path.isabs(root): 126 return None 127 128 abspath = os.path.abspath(path) 129 mydebug('relpath: abspath(%s) = %s', path, abspath) 130 131 # make sure the root and abspath both end with a '/' or '\' 132 if sys.platform == 'win32': 133 slash = '\\' 134 else: 135 slash = '/' 136 137 if not root[-1]==slash: 138 root += slash 139 if not abspath[-1]==slash: 140 abspath += slash 141 142 mydebug('relpath: root = %s', root) 143 144 #print root, abspath 145 if not abspath[:len(root)]==root: 146 #print abspath[:len(root)], root 147 return None 148 mydebug('relpath: relpath = %s',abspath[len(root):]) 149 return abspath[len(root):] 150 151 def my_abspath(path): 152 """expand path including shell variables and homedir 153 to the absolute path 154 """ 155 return os.path.abspath(os.path.expanduser(os.path.expandvars(path))) 156 157 def update_follow(path): 158 """ tries to find a follow directive that matches path 159 and if path refers to a symbolic link the real path of the symbolic 160 link is returned. """ 161 try: 162 op.symlinks 163 except AttributeError: 164 make_symlinks() 165 rpath = relpath(op.root, path) 166 mydebug('update_follow: rpath %s', rpath) 167 result = None 168 foll = None 169 for k in op.symlinks: 170 v = op.symlinks[k] 171 if v==rpath: 172 result = os.path.realpath(os.path.abspath(path)) 173 foll = v 174 mydebug('update_follow: link %s, real %s',v,result) 175 break 176 if result: 177 op.symlinks[result] = foll 178 179 return result, foll 180 181 def conf_parser(conffilepath, delimiter = '=', dic = {}): 182 """parse the unison configuration file at conffilename and populate a dictionary 183 with configuration options from there. If dic is a dictionary, these options are added to this 184 one (can be used to recursively call this function for include statements).""" 185 try: 186 conffile = open(conffilepath,'r') 187 except IOError: 188 mydebug('could not open configuration file at %s',conffilepath) 189 return None 190 191 res = dic 192 193 for line in conffile: 194 line = line.strip() 195 if len(line)<1 or line[0]=='#': 196 continue 197 elif line.startswith('include'): 198 dn = os.path.dirname(conffilepath) 199 fn = line.split()[1].strip() 200 conf_parser(os.path.join(dn,fn), dic = res) 201 else: 202 k,v=[s.strip() for s in line.split('=',1)] 203 if res.has_key(k): 204 res[k].append(v) 205 else: 206 res[k]=[v] 207 return res 208 209 ################################################ 210 # Linux specific code here 211 ################################################ 212 if sys.platform.startswith('linux'): 213 import pyinotify 214 215 class HandleEvents(pyinotify.ProcessEvent): 216 wm = None 217 218 #def process_IN_CREATE(self, event): 219 # print "Creating:", event.pathname 220 221 #def process_IN_DELETE(self, event): 222 # print "Removing:", event.pathname 223 224 #def process_IN_MODIFY(self, event): 225 # print "Modifying:", event.pathname 226 227 # def process_IN_MOVED_TO(self, event): 228 # print "Moved to:", event.pathname 229 230 # def process_IN_MOVED_FROM(self, event): 231 # print "Moved from:", event.pathname 232 233 # def process_IN_ATTRIB(self, event): 234 # print "attributes:", event.pathname 235 236 def process_default(self, event): 237 mydebug('process_default: event %s', event) 238 # code for adding dirs is obsolete since there is the auto_add option 239 # if event.dir: 240 # if event.mask&pyinotify.IN_CREATE: 241 # print 'create:', event.pathname , self.add_watch(event.pathname,rec=True) 242 # elif event.mask&pyinotify.IN_DELETE: 243 # print 'remove', event.pathname, self.remove_watch(event.pathname) 244 # pass 245 # elif event.mask&pyinotify.IN_MOVED_FROM: 246 # print 'move from', event.pathname, self.remove_watch(event.pathname, rec=True) 247 # pass 248 # elif event.mask&pyinotify.IN_MOVED_TO: 249 # print 'move to', event.pathname, self.add_watch(event.pathname,rec=True) 250 # else: 251 # pass 252 #handle creation of links that should be followed 253 if os.path.islink(event.pathname): 254 #special handling for links 255 mydebug('process_default: link %s created/changed. Checking for follows', event.pathname) 256 p, l = update_follow(event.pathname) 257 if p: 258 self.add_watch(p,rec=True,auto_add=True) 259 mydebug('process_default: follow link %s to %s',l,p) 260 #TODO: should handle deletion of links that are followed (delete the respective watches) 261 update_changes([event.pathname]) 262 263 def remove_watch(self, pathname, **kwargs): 264 if self.watches.has_key(pathname): 265 return self.wm.rm_watch(self.watches.pop(pathname),**kwargs) 266 return None 267 268 def add_watch(self, pathname, **kwargs): 269 neww = self.wm.add_watch(pathname, self.mask, **kwargs) 270 self.watches.update(neww) 271 return neww 272 273 def init_watches(self, abspaths, follows): 274 self.watches = {} 275 for abspath in abspaths: 276 self.watches.update(self.wm.add_watch(abspath,self.mask,rec=True,auto_add=True)) 277 #we have to add watches for follow statements since pyinotify does 278 #not do recursion across symlinks 279 make_symlinks() 280 for link in op.symlinks: 281 mydebug('following symbolic link %s',link) 282 if not self.watches.has_key(link): 283 self.watches.update(self.wm.add_watch(link,self.mask,rec=True,auto_add=True)) 284 285 mydebug('init_watches: added paths %s\n based on paths %s\n and follows %s',self.watches,op.abspaths, op.follow) 286 287 288 def linuxwatcher(): 289 p = HandleEvents() 290 wm = pyinotify.WatchManager() # Watch Manager 291 p.wm = wm 292 p.mask = pyinotify.IN_CREATE | pyinotify.IN_DELETE | pyinotify.IN_MODIFY | pyinotify.IN_ATTRIB | pyinotify.IN_MOVED_TO | pyinotify.IN_MOVED_FROM # watched events 293 294 notifier = pyinotify.Notifier(wm, p) 295 p.init_watches(op.abspaths, op.follow) 296 notifier.loop() 297 298 299 ################################################# 300 # END Linux specific code 301 ################################################# 302 303 ################################################# 304 # MacOsX specific code 305 ################################################# 306 if sys.platform == 'darwin': 307 from FSEvents import * 308 import objc 309 310 def filelevel_approx(path): 311 """in order to avoid scanning the entire directory including sub 312 directories by unison, we have to say which files have changed. Because 313 this is a stupid program it only checks modification times within the 314 update interval. in case there are no files modified in this interval, 315 the entire directory is listed. 316 A deleted file can not be found like this. Therefore also deletes will 317 trigger a rescan of the directory (including subdirs) 318 319 The impact of rescans could be limited if one could make 320 unison work nonrecursively. 321 """ 322 result = [] 323 #make a list of all files in question (all files in path w/o dirs) 324 try: 325 names = os.listdir(path) 326 except os.error: 327 #path does not exist (anymore?). Add it to the results 328 mydebug("adding nonexisting path %s for sync",path) 329 result.append(path) 330 names = None 331 332 if names: 333 for nm in names: 334 full_path = os.path.join(path,nm) 335 st = os.lstat(full_path) 336 #see if the dir it was modified recently 337 if st.st_mtime>time()-float(op.latency): 338 result.append(full_path) 339 340 if result == []: 341 result.append(path) 342 343 return result 344 345 346 def fsevents_callback(streamRef, clientInfo, numEvents, eventPaths, eventMasks, eventIDs): 347 mydebug("fsevents_callback(streamRef = %s, clientInfo = %s, numEvents = %s)", streamRef, clientInfo, numEvents) 348 mydebug("fsevents_callback: FSEventStreamGetLatestEventId(streamRef) => %s", FSEventStreamGetLatestEventId(streamRef)) 349 mydebug("fsevents_callback: eventpaths = %s",eventPaths) 350 351 full_path = clientInfo 352 353 result = [] 354 for i in range(numEvents): 355 path = eventPaths[i] 356 if path[-1] == '/': 357 path = path[:-1] 358 359 if eventMasks[i] & kFSEventStreamEventFlagMustScanSubDirs: 360 recursive = True 361 362 elif eventMasks[i] & kFSEventStreamEventFlagUserDropped: 363 mymesg("BAD NEWS! We dropped events.") 364 mymesg("Forcing a full rescan.") 365 recursive = 1 366 path = full_path 367 368 elif eventMasks[i] & kFSEventStreamEventFlagKernelDropped: 369 mymesg("REALLY BAD NEWS! The kernel dropped events.") 370 mymesg("Forcing a full rescan.") 371 recursive = 1 372 path = full_path 373 374 else: 375 recursive = False 376 377 #now we should know what to do: build a file directory list 378 #I assume here, that unison takes a flag for recursive scans 379 #JV: commented out (not implemented by Unison) 380 # if recursive: 381 # #we have to check all subdirectories 382 # if isinstance(path,list): 383 # #we have to check all base paths 384 # allpathsrecursive = [p + '\tr'] 385 # result.extend(path) 386 # else: 387 # result.append(path+'\tr') 388 # else: 389 #just add the path 390 #result.append(path) 391 #try to find out what has changed 392 result.extend(filelevel_approx(path)) 393 394 mydebug('Dirs sent: %s',eventPaths) 395 #TODO: handle creation/deletion of links that should be followed 396 update_changes(result) 397 398 try: 399 f = open(op.absstatus,'w') 400 f.write('last_item = %d'%eventIDs[-1]) 401 f.close() 402 except IOError: 403 mymesg('failed to open status file %s', op.absstatus) 404 405 def my_FSEventStreamCreate(paths): 406 mydebug('my_FSEventStreamCreate: selected paths are: %s',paths) 407 408 if op.sinceWhen == 'now': 409 op.sinceWhen = kFSEventStreamEventIdSinceNow 410 411 try: 412 op.symlinks 413 except AttributeError: 414 make_symlinks() 415 416 for sl in op.symlinks: 417 #check if that path is already there 418 found=False 419 ln = op.symlinks[sl] 420 for path in paths: 421 if relpath(op.root,path)==ln: 422 found = True 423 break 424 if not found: 425 mydebug('my_FSEventStreamCreate: watch followed link %s',ln) 426 paths.append(os.path.join(op.root,ln)) 427 428 streamRef = FSEventStreamCreate(kCFAllocatorDefault, 429 fsevents_callback, 430 paths, #will this pass properly through? yes it does. 431 paths, 432 int(op.sinceWhen), 433 float(op.latency), 434 int(op.flags)) 435 if streamRef is None: 436 mymesg("ERROR: FSEVentStreamCreate() => NULL") 437 return None 438 439 if op.verbose: 440 FSEventStreamShow(streamRef) 441 442 #print ('my_FSE', streamRef) 443 444 return streamRef 445 446 def macosxwatcher(): 447 #since when? if it is 'now' try to read state 448 if op.sinceWhen == 'now': 449 di = conf_parser(op.absstatus) 450 if di and di.has_key('last_item'): 451 #print di['last_item'][-1] 452 op.sinceWhen = di['last_item'][-1] 453 #print op.sinceWhen 454 455 streamRef = my_FSEventStreamCreate(op.abspaths) 456 #print streamRef 457 if streamRef is None: 458 print('failed to get a Stream') 459 exit(1) 460 461 FSEventStreamScheduleWithRunLoop(streamRef, CFRunLoopGetCurrent(), kCFRunLoopDefaultMode) 462 463 startedOK = FSEventStreamStart(streamRef) 464 if not startedOK: 465 print("failed to start the FSEventStream") 466 exit(1) 467 468 if op.flush_seconds >= 0: 469 mydebug("CFAbsoluteTimeGetCurrent() => %.3f", CFAbsoluteTimeGetCurrent()) 470 471 timer = CFRunLoopTimerCreate(None, 472 CFAbsoluteTimeGetCurrent() + float(op.flush_seconds), 473 float(op.flush_seconds), 474 0, 0, timer_callback, streamRef) 475 CFRunLoopAddTimer(CFRunLoopGetCurrent(), timer, kCFRunLoopDefaultMode) 476 477 try: 478 CFRunLoopRun() 479 except KeyboardInterrupt: 480 mydebug('stop called via Keyboard, cleaning up.') 481 #Stop / Invalidate / Release 482 FSEventStreamStop(streamRef) 483 FSEventStreamInvalidate(streamRef) 484 FSEventStreamRelease(streamRef) 485 mydebug('FSEventStream closed') 486 487 ################################################# 488 # END MacOsX specific code 489 ################################################# 490 491 ################################################# 492 # Windows specific code 493 ################################################# 494 if sys.platform == 'win32': 495 import win32file 496 import win32con 497 498 FILE_LIST_DIRECTORY = 0x0001 499 500 def win32watcherThread(abspath,file_lock): 501 dirHandle = win32file.CreateFile ( 502 abspath, 503 FILE_LIST_DIRECTORY, 504 win32con.FILE_SHARE_READ | win32con.FILE_SHARE_WRITE, 505 None, 506 win32con.OPEN_EXISTING, 507 win32con.FILE_FLAG_BACKUP_SEMANTICS, 508 None 509 ) 510 while 1: 511 results = win32file.ReadDirectoryChangesW ( 512 dirHandle, 513 1024, 514 True, 515 win32con.FILE_NOTIFY_CHANGE_FILE_NAME | 516 win32con.FILE_NOTIFY_CHANGE_DIR_NAME | 517 win32con.FILE_NOTIFY_CHANGE_ATTRIBUTES | 518 win32con.FILE_NOTIFY_CHANGE_SIZE | 519 win32con.FILE_NOTIFY_CHANGE_LAST_WRITE | 520 win32con.FILE_NOTIFY_CHANGE_SECURITY, 521 None, 522 None 523 ) 524 for action, file in results: 525 full_filename = os.path.join (abspath, file) 526 # This will return 'dir updated' for every file update within dir, but 527 # we don't want to send unison on a full dir sync in this situation. 528 if not (os.path.isdir(full_filename) and action == 3): 529 file_lock.acquire() 530 update_changes_nomangle(full_filename) 531 file_lock.release() 532 533 def win32watcher(): 534 file_lock = threading.Lock() 535 threads = [ threading.Thread(target=win32watcherThread,args=(abspath,file_lock,)) for abspath in op.abspaths ] 536 for thread in threads: 537 thread.setDaemon(True) 538 thread.start() 539 540 try: 541 while 1: 542 sleep(3600) 543 except KeyboardInterrupt: 544 print("Cleaning up.") 545 546 ################################################# 547 # END Windows specific code 548 ################################################# 549 550 if __name__=='__main__': 551 global op 552 553 usage = """usage: %prog [options] root [path] [path]... 554 This program monitors file system changes on all given (relative to root) paths 555 and dumps paths (relative to root) files to a file. When launched, this file is 556 recreated. While running new events are added. This can be read by UNISON 557 to trigger a sync on these files. If root is a valid unison profile, we attempt 558 to read all the settings from there.""" 559 560 parser = OptionParser(usage=usage) 561 parser.add_option("-w", "--sinceWhen", dest="sinceWhen", 562 help="""starting point for filesystem updates to be captured 563 Defaults to 'now' in the first run 564 or the last captured change""",default = 'now', metavar="SINCEWHEN") 565 parser.add_option("-l", "--latency", dest="latency", 566 help="set notification LATENCY in seconds. default 5",default = 5, metavar="LATENCY") 567 parser.add_option("-f", "--flags", dest="flags", 568 help="(macosx) set flags (who knows what they mean. defaults to 0",default = 0, metavar="FLAGS") 569 parser.add_option("-s", "--flushseconds", dest="flush_seconds", 570 help="(macosx) TIME interval in second until flush is forced. values < 0 turn it off. ",default = 1, metavar="TIME") 571 parser.add_option("-o", "--outfile", dest="outfile", 572 help="location of the output file. Defaults to UPATH/changes",default = 'changes', metavar="PATH") 573 parser.add_option("-t", "--statefile", dest="statefile", 574 help="(macosx) location of the state file (absolute or relative to UPATH). Defaults to UPATH/state",default = 'state', metavar="PATH") 575 parser.add_option("-u", "--unisonconfig", dest="uconfdir", 576 help='path to the unison config directory. default ~/.unison', 577 default = '~/.unison', metavar = 'UPATH') 578 parser.add_option("-z", "--follow", dest="follow", 579 help="define a FOLLOW directive. This is equivalent to the -follow option in unison \ 580 (except that for now only 'Paths' are supported). This option can appear multiple times. \ 581 if a unison configuration file is loaded, it takes precedence over this option", 582 action='append',metavar = 'FOLLOW') 583 parser.add_option("-q", "--quiet", 584 action="store_false", dest="verbose", default=True, 585 help="don't print status messages to stdout") 586 587 parser.add_option("-d", "--debug", 588 action="store_true", dest="debug", default=False, 589 help="print debug messages to stderr") 590 591 592 (op, args) = parser.parse_args() 593 594 595 if len(args)<1: 596 parser.print_usage() 597 sys.exit() 598 599 #other paths 600 op.absuconfdir = my_abspath(op.uconfdir) 601 op.absstatus = os.path.join(op.absuconfdir,op.statefile) 602 op.absoutfile = os.path.join(op.absuconfdir,op.outfile) 603 604 605 #figure out if the root argument is a valid configuration file name 606 p = args[0] 607 fn = '' 608 if os.path.exists(p) and not os.path.isdir(p): 609 fn = p 610 elif os.path.exists(os.path.join(op.absuconfdir,p)): 611 fn = os.path.join(op.absuconfdir,p) 612 op.unison_conf = conf_parser(fn) 613 614 #now check for the relevant information 615 root = None 616 paths = None 617 if op.unison_conf and op.unison_conf.has_key('root'): 618 #find the local root 619 root = None 620 paths = None 621 for r in op.unison_conf['root']: 622 if r[0]=='/': 623 root = r 624 if op.unison_conf.has_key('path'): 625 paths = op.unison_conf['path'] 626 if op.unison_conf and op.unison_conf.has_key('follow'): 627 op.follow = op.unison_conf['follow'] 628 else: 629 #see if follows were defined 630 try: 631 op.follow 632 except AttributeError: 633 op.follow = [] 634 635 if not root: 636 #no root up to here. get it from args 637 root = args[0] 638 639 if not paths: 640 paths = args[1:] 641 642 #absolute paths 643 op.root = my_abspath(root) 644 op.abspaths = [os.path.join(root,path) for path in paths] 645 if op.abspaths == []: 646 #no paths specified -> make root the path to observe 647 op.abspaths = [op.root] 648 #print op.root 649 #print op.abspaths 650 651 mydebug('options: %s',op) 652 mydebug('arguments: %s',args) 653 654 #cleaning up the change file 655 try: 656 f=open(op.absoutfile,'w') 657 f.close() 658 except IOError: 659 mymesg('failed to open output file. STOP.') 660 exit(1) 661 662 #stop watching when stdin is closed 663 def exitThread(): 664 while sys.stdin.readline(): pass 665 os._exit(0) 666 t = threading.Thread(target=exitThread) 667 t.daemon = True 668 t.start() 669 670 if sys.platform=='darwin': 671 macosxwatcher() 672 elif sys.platform.startswith('linux'): 673 linuxwatcher() 674 elif sys.platform.startswith('win32'): 675 win32watcher() 676 else: 677 mymesg('unsupported platform %s',sys.platform)