Difference between revisions of "Find orphans.py"

From MythTV Official Wiki
Jump to: navigation, search
m (update after recent changes, and no need to mention the script that shall not be named)
m (Undo revision 56430 by Davemc (talk) it already says so right at the top.)
(11 intermediate revisions by 3 users not shown)
Line 3: Line 3:
 
|author=Raymond Wagner
 
|author=Raymond Wagner
 
|short=orphaned file/recording scanner
 
|short=orphaned file/recording scanner
|long=A scanner to look for missing and unknown recording files. This is informative only, informing the user of the files but taking no action.
+
|long=A scanner to look for missing and unknown recording files. This will only delete files after multiple confirmations.
 
|category=Maintenance
 
|category=Maintenance
 
|file=find_orphans.py
 
|file=find_orphans.py
|S24=yes}}
+
|S24=yes|S25=yes}}
  
 
This script shows recordings with missing files, or files with missing recordings.  It can handle multiple backends, and does not need to be run locally, however recordings stored on offline backends will be marked as orphaned.
 
This script shows recordings with missing files, or files with missing recordings.  It can handle multiple backends, and does not need to be run locally, however recordings stored on offline backends will be marked as orphaned.
Line 49: Line 49:
 
#!/usr/bin/env python
 
#!/usr/bin/env python
  
from MythTV import MythDB, MythBE, Recorded
+
from MythTV import MythDB, MythBE, Recorded, MythError
 
from socket import timeout
 
from socket import timeout
  
Line 72: Line 72:
 
         self.size = int(size)
 
         self.size = int(size)
 
     def pprint(self):
 
     def pprint(self):
         name = '%s: %s' % (self.host, os.path.join(self.path, self))
+
         name = u'%s: %s' % (self.host, os.path.join(self.path, self))
         print '  {0:<90}{1:>8}'.format(name, human_size(self.size))
+
         print u'  {0:<90}{1:>8}'.format(name, human_size(self.size))
 
     def delete(self):
 
     def delete(self):
 
         be = MythBE(self.host, db=DB)
 
         be = MythBE(self.host, db=DB)
Line 81: Line 81:
 
     _table = 'recorded'
 
     _table = 'recorded'
 
     def pprint(self):
 
     def pprint(self):
         name = '{0.hostname}: {0.title}'.format(self)
+
         name = u'{0.hostname}: {0.title}'.format(self)
 
         if self.subtitle:
 
         if self.subtitle:
             name += ' - '+self.subtitle
+
             name += u' - '+self.subtitle
         print '  {0:<70}{1:>28}'.format(name,self.basename)
+
         print u'  {0:<70}{1:>28}'.format(name,self.basename)
  
 
def printrecs(title, recs):
 
def printrecs(title, recs):
Line 90: Line 90:
 
     for rec in sorted(recs, key=lambda x: x.title):
 
     for rec in sorted(recs, key=lambda x: x.title):
 
         rec.pprint()
 
         rec.pprint()
     print '{0:>88}{1:>12}'.format('Count:',len(recs))
+
     print u'{0:>88}{1:>12}'.format('Count:',len(recs))
  
 
def printfiles(title, files):
 
def printfiles(title, files):
Line 97: Line 97:
 
         f.pprint()
 
         f.pprint()
 
     size = sum([f.size for f in files])
 
     size = sum([f.size for f in files])
     print '{0:>88}{1:>12}'.format('Total:',human_size(size))
+
     print u'{0:>88}{1:>12}'.format('Total:',human_size(size))
  
 
def populate(host=None):
 
def populate(host=None):
Line 181: Line 181:
 
             else:
 
             else:
 
                 res = raw_input("'yes' or 'no' > ")
 
                 res = raw_input("'yes' or 'no' > ")
 +
    except MythError:
 +
        name = u'{0.hostname}: {0.title}'.format(rec)
 +
        if rec.subtitle:
 +
            name += ' - '+rec.subtitle
 +
        print "Warning: Failed to delete '" + name + "'"
 
     except KeyboardInterrupt:
 
     except KeyboardInterrupt:
 
         pass
 
         pass
Line 223: Line 228:
  
 
         opts = []
 
         opts = []
#        if len(recs):
+
        if len(recs):
#            opts.append(['Delete orphaned recording entries', delete_recs, recs])
+
            opts.append(['Delete orphaned recording entries', delete_recs, recs])
 
         if len(zerorecs):
 
         if len(zerorecs):
 
             opts.append(['Delete zero byte recordings', delete_recs, zerorecs])
 
             opts.append(['Delete zero byte recordings', delete_recs, zerorecs])
#        if len(orphvids):
+
        if len(orphvids):
#            opts.append(['Delete orphaned video files', delete_files, orphvids])
+
            opts.append(['Delete orphaned video files', delete_files, orphvids])
 
         if len(orphimgs):
 
         if len(orphimgs):
 
             opts.append(['Delete orphaned snapshots', delete_files, orphimgs])
 
             opts.append(['Delete orphaned snapshots', delete_files, orphimgs])
Line 265: Line 270:
 
BE = MythBE(db=DB)
 
BE = MythBE(db=DB)
 
DB.searchRecorded.handler = MyRecorded
 
DB.searchRecorded.handler = MyRecorded
 +
DB.searchRecorded.dbclass = MyRecorded
  
 
if __name__ == '__main__':
 
if __name__ == '__main__':
Line 274: Line 280:
  
 
[[Category:Python_Scripts]]
 
[[Category:Python_Scripts]]
 +
[[Category:Management Scripts]]

Revision as of 13:32, 4 October 2012

Important.png Note: The correct title of this article is find_orphans.py. It appears incorrectly here due to technical restrictions.


Author Raymond Wagner
Description A scanner to look for missing and unknown recording files. This will only delete files after multiple confirmations.
Supports Version24.png  Version25.png  


This script shows recordings with missing files, or files with missing recordings. It can handle multiple backends, and does not need to be run locally, however recordings stored on offline backends will be marked as orphaned.

Additionally, this allows listing of database backups, and the listing and deletion of zero byte recordings, orphaned snapshots, and other unknown files.

>./find_orphans.py
Recordings with missing files
  Undercovers - Devices                  4642_20101006201300.mpg

Orphaned video files
  mythbe:/srv/mounts/twotb_1/video/2054_20080225110000.mpg                          2.5GB
                                                                          Total:    2.5GB

Orphaned snapshots
  myth0:/srv/mounts/myth0_1/video/4122_20101013113500.mpg.png                       2.6KB
  mythbe:/srv/mounts/twotb_1/video/2029_20100409024900.mpg.png                     84.9KB
  mythbe:/srv/mounts/twotb_1/video/2047_20100807180500.mpg.png                     92.9KB
  mythbe:/srv/mounts/twotb_1/video/2059_20100630090000.mpg.png                     87.0KB
                                                                          Total:  267.4KB

Database backups
  mythbe:/mnt/mythtv/store/backups/mythconverg--20101007134000.sql                 17.3MB
  mythbe:/mnt/mythtv/store/backups/mythconverg-1254-20100902174922.sql.gz          13.1MB
  mythbe:/mnt/mythtv/store/backups/mythconverg-1263-20100913163154.sql             62.1MB
  mythbe:/mnt/mythtv/store/backups/mythconverg-1263-20100913163216.sql.gz          13.0MB
  mythbe:/mnt/mythtv/store/backups/mythconverg-1263-20101007134659.sql.gz          15.9MB
  mythbe:/mnt/mythtv/store/backups/mythconverg-1264-20101008023651.sql.gz          16.5MB
                                                                          Total:  137.9MB

Other files
  mythbe:/srv/mounts/twotb_1/video/4121_20100312215900.mpg.tmp                    398.6MB
  mythbe:/srv/mounts/twotb_1/video/4191_20090928200000.mpg.tmp                      2.4GB
  mythbe:/srv/mounts/twotb_1/video/4191_20091005195900.mpg.tmp                      2.6GB
  mythbe:/srv/mounts/twotb_1/video/4642_20101006201300.mpg.1                        4.9GB
                                                                          Total:   10.2GB


PythonIcon.png find_orphans.py

#!/usr/bin/env python

from MythTV import MythDB, MythBE, Recorded, MythError
from socket import timeout

import os
import sys

def human_size(s):
    s = float(s)
    o = 0
    while s > 1000:
        s /= 1000
        o += 1
    return str(round(s,1))+('B ','KB','MB','GB')[o]

class File( str ):
    def __new__(self, host, group, path, name, size):
        return str.__new__(self, name)
    def __init__(self, host, group, path, name, size):
        self.host = host
        self.group = group
        self.path = path
        self.size = int(size)
    def pprint(self):
        name = u'%s: %s' % (self.host, os.path.join(self.path, self))
        print u'  {0:<90}{1:>8}'.format(name, human_size(self.size))
    def delete(self):
        be = MythBE(self.host, db=DB)
        be.deleteFile(self, self.group)

class MyRecorded( Recorded ):
    _table = 'recorded'
    def pprint(self):
        name = u'{0.hostname}: {0.title}'.format(self)
        if self.subtitle:
            name += u' - '+self.subtitle
        print u'  {0:<70}{1:>28}'.format(name,self.basename)

def printrecs(title, recs):
    print title
    for rec in sorted(recs, key=lambda x: x.title):
        rec.pprint()
    print u'{0:>88}{1:>12}'.format('Count:',len(recs))

def printfiles(title, files):
    print title
    for f in sorted(files, key=lambda x: x.path):
        f.pprint()
    size = sum([f.size for f in files])
    print u'{0:>88}{1:>12}'.format('Total:',human_size(size))

def populate(host=None):
    unfiltered = []
    kwargs = {'livetv':True}
    if host:
        with DB as c:
            c.execute("""SELECT count(1) FROM settings
                         WHERE hostname=%s AND value=%s""",
                        (host, 'BackendServerIP'))
            if c.fetchone()[0] == 0:
                raise Exception('Invalid hostname specified on command line.')
        hosts = [host]
        kwargs['hostname'] = host
    else:
        with DB as c:
            c.execute("""SELECT hostname FROM settings
                         WHERE value='BackendServerIP'""")
            hosts = [r[0] for r in c.fetchall()]
    for host in hosts:
        for sg in DB.getStorageGroup():
            if sg.groupname in ('Videos','Banners','Coverart',\
                                'Fanart','Screenshots','Trailers'):
                continue
            try:
                dirs,files,sizes = BE.getSGList(host, sg.groupname, sg.dirname)
                for f,s in zip(files,sizes):
                    newfile = File(host, sg.groupname, sg.dirname, f, s)
                    if newfile not in unfiltered:
                        unfiltered.append(newfile)
            except:
                pass

    recs = list(DB.searchRecorded(**kwargs))

    zerorecs = []
    orphvids = []
    for rec in list(recs):
        if rec.basename in unfiltered:
            recs.remove(rec)
            i = unfiltered.index(rec.basename)
            f = unfiltered.pop(i)
            if f.size < 1024:
                zerorecs.append(rec)
            name = rec.basename.rsplit('.',1)[0]
            for f in list(unfiltered):
                if name in f:
                    unfiltered.remove(f)
    for f in list(unfiltered):
        if not (f.endswith('.mpg') or f.endswith('.nuv')):
            continue
        orphvids.append(f)
        unfiltered.remove(f)

    orphimgs = []
    for f in list(unfiltered):
        if not f.endswith('.png'):
            continue
        orphimgs.append(f)
        unfiltered.remove(f)

    dbbackup = []
    for f in list(unfiltered):
        if 'sql' not in f:
            continue
        dbbackup.append(f)
        unfiltered.remove(f)

    return (recs, zerorecs, orphvids, orphimgs, dbbackup, unfiltered)

def delete_recs(recs):
    printrecs('The following recordings will be deleted', recs)
    print 'Are you sure you want to continue?'
    try:
        res = raw_input('> ')
        while True:
            if res == 'yes':
                for rec in recs:
                    rec.delete(True, True)
                break
            elif res == 'no':
                break
            else:
                res = raw_input("'yes' or 'no' > ")
    except MythError:
        name = u'{0.hostname}: {0.title}'.format(rec)
        if rec.subtitle:
            name += ' - '+rec.subtitle
        print "Warning: Failed to delete '" + name + "'"
    except KeyboardInterrupt:
        pass
    except EOFError:
        sys.exit(0)

def delete_files(files):
    printfiles('The following files will be deleted', files)
    print 'Are you sure you want to continue?'
    try:
        res = raw_input('> ')
        while True:
            if res == 'yes':
                for f in files:
                    f.delete()
                break
            elif res == 'no':
                break
            else:
                res = raw_input("'yes' or 'no' > ")
    except KeyboardInterrupt:
        pass
    except EOFError:
        sys.exit(0)

def main(host=None):
   while True:
        recs, zerorecs, orphvids, orphimgs, dbbackup, unfiltered = populate(host)

        if len(recs):
            printrecs("Recordings with missing files", recs)
        if len(zerorecs):
            printrecs("Zero byte recordings", zerorecs)
        if len(orphvids):
            printfiles("Orphaned video files", orphvids)
        if len(orphimgs):
            printfiles("Orphaned snapshots", orphimgs)
        if len(dbbackup):
            printfiles("Database backups", dbbackup)
        if len(unfiltered):
            printfiles("Other files", unfiltered)

        opts = []
        if len(recs):
            opts.append(['Delete orphaned recording entries', delete_recs, recs])
        if len(zerorecs):
            opts.append(['Delete zero byte recordings', delete_recs, zerorecs])
        if len(orphvids):
            opts.append(['Delete orphaned video files', delete_files, orphvids])
        if len(orphimgs):
            opts.append(['Delete orphaned snapshots', delete_files, orphimgs])
        if len(unfiltered):
            opts.append(['Delete other files', delete_files, unfiltered])
        opts.append(['Refresh list', None, None])
        print 'Please select from the following'
        for i, opt in enumerate(opts):
            print ' {0}. {1}'.format(i+1, opt[0])

        try:
            inner = True
            res = raw_input('> ')
            while inner:
                try:
                    res = int(res)
                except:
                    res = raw_input('input number. ctrl-c to exit > ')
                    continue
                if (res <= 0) or (res > len(opts)):
                    res = raw_input('input number within range > ')
                    continue
                break
            opt = opts[res-1]
            if opt[1] is None:
                continue
            else:
                opt[1](opt[2])

        except KeyboardInterrupt:
            break
        except EOFError:
            sys.exit(0)

DB = MythDB()
BE = MythBE(db=DB)
DB.searchRecorded.handler = MyRecorded
DB.searchRecorded.dbclass = MyRecorded

if __name__ == '__main__':
    if len(sys.argv) == 2:
        main(sys.argv[1])
    else:
        main()