Difference between revisions of "MythCompress"
m (wiki formatting) |
m (additional wiki formatting) |
||
(3 intermediate revisions by 2 users not shown) | |||
Line 1: | Line 1: | ||
Due to disc restrictions in my mythbox i decided to write a script to compress all my videos. | Due to disc restrictions in my mythbox i decided to write a script to compress all my videos. | ||
Mythtranscode does this but the videos have been recorded quite some time ago and are spread over multiple discs. Compressing those videos to a single destination is no option because they probably won't fit onto one partition. Therefore i wrote a script that dynamically picks a show that is at least two weeks old, compresses it and saves it to a partition with enough disc space. | Mythtranscode does this but the videos have been recorded quite some time ago and are spread over multiple discs. Compressing those videos to a single destination is no option because they probably won't fit onto one partition. Therefore i wrote a script that dynamically picks a show that is at least two weeks old, compresses it and saves it to a partition with enough disc space. | ||
− | So, there are multiple folders where videos are waiting for compression and multiple folders where compressed videos are stored. Actually, these can be the same, but they don't have to be as long they are visible to mythtv (see mythtv-setup and storing directories). For compressed videos i created another storing group called 'compressed' and added all destination folders. Additionally i wanted this script to run as a system service and between 1 am and 6 pm. | + | So, there are multiple folders where videos are waiting for compression and multiple folders where compressed videos are stored. Actually, these folders can be the same, but they don't have to be as long they are visible to mythtv (see mythtv-setup and storing directories). For compressed videos i created another storing group called 'compressed' and added all destination folders. Videos are recorded in two different formats by my tv-card (PAL and 720p). This script decides upon video resolution which parameters to pick. Additionally i wanted this script to run as a system service and between 1 am and 6 pm. |
Processing: | Processing: | ||
Line 12: | Line 12: | ||
# Back to 1. | # Back to 1. | ||
+ | Following programs are required: | ||
+ | python 2.7, ffmpeg (ffmpeg version git-2013-10-01-2e2a2d8), ffprobe (comes with ffmpeg) | ||
− | This script consists of three | + | This script consists of three python scripts: mythCompress.py, mythLogging.py, mythFunctions.py |
− | and one system script: compressVideos | + | and one system (init.d) script: compressVideos |
− | + | '''mythCompress.py''' | |
− | + | {{python|mythCompress.py| | |
− | + | <pre> | |
− | + | #!/usr/bin/env python | |
− | + | ||
− | + | import MySQLdb | |
− | + | import os, sys, glob | |
− | + | import shlex, subprocess | |
− | + | import tempfile | |
− | + | import time | |
− | + | from datetime import datetime, timedelta | |
− | + | from os.path import basename | |
− | + | import re | |
− | + | import shutil | |
− | + | import math | |
− | + | import atexit | |
− | + | ||
− | + | from mythFunctions import * | |
− | + | from mythLogging import error, debug, clean | |
− | + | ||
− | + | import psutil | |
− | + | os.nice(19) | |
− | + | program = psutil.Process(os.getpid()) | |
− | + | program.set_ionice(psutil.IOPRIO_CLASS_IDLE) | |
− | + | ||
− | + | ###################################################################### | |
− | + | # SETTINGS | |
− | + | # | |
− | + | # folders to save compressed videos | |
− | + | DESTINATION=[ "/media/myth/compressed/" , "/media/data/mythtv/compressed/" ] | |
− | + | # folders to read uncompressed videos | |
− | + | SOURCES=[ "/media/data/mythtv/default/" , "/media/myth/default/" ] | |
− | + | TEMPPARENTDIR=os.path.expanduser("~/mythtv-tmp/") | |
− | + | PRINTDEBUG=True | |
− | + | mysql_opts = { | |
− | + | 'host': "localhost", | |
− | + | 'user': "mythtv", | |
− | + | 'pass': "CHANGEME", | |
− | + | 'db': "mythconverg" | |
− | + | } | |
− | + | minimumAge = timedelta(days=14) # for conversion | |
− | + | ffmpegBinary="/home/rfs/bin/ffmpeg" | |
− | + | ffprobeBinary="/home/rfs/bin/ffprobe" | |
− | + | ||
− | + | freeSpaceRequired=20000000000 | |
− | + | freeSpaceInTempRequired=10000000000 | |
− | + | ###################################################################### | |
− | + | # constants | |
− | + | PID=os.getpgid(0) | |
− | + | now=datetime.now() | |
− | + | ||
− | + | ################################################################# | |
− | + | # functions | |
− | + | def createTemp(status, parentDirectory): | |
− | + | tempDir=None | |
− | + | if os.path.exists(parentDirectory): | |
− | + | tempDir = tempfile.mkdtemp(prefix=parentDirectory + str(PID) + "-") | |
− | + | debug(status, "Temporary directory: " + tempDir) | |
− | + | else: | |
− | + | status.tempDir=None | |
− | + | error(status, "Parent directory for temp does not exist.") | |
− | + | # change directory to working directory | |
− | + | try: | |
− | + | os.chdir(tempDir) | |
− | + | except OSError: | |
− | + | status.tempDir=None | |
− | + | error(status, "Could not find temporary directory (OSError).") | |
− | + | # return result | |
− | + | return tempDir | |
− | + | ||
− | + | class status(object): | |
− | + | prefix=None | |
− | + | tempDir=None | |
− | + | ||
− | + | class videoFile(object): | |
− | + | channelId=None | |
− | + | channelStart=None | |
− | + | channelDate=None | |
− | + | title=None | |
− | + | ||
− | + | # quellvideo | |
− | + | filename=None | |
− | + | basename=None | |
− | + | videoInfo=None | |
− | + | duration=None | |
− | + | numAudioStreams=None | |
− | + | ||
− | + | # temporaere ausgabe | |
− | + | tmpFilename=None | |
− | + | tmpBasename=None | |
− | + | ||
− | + | # ziel | |
− | + | newFilename=None | |
− | + | newBasename=None | |
− | + | newVideoInfo=None | |
− | + | newDuration=None | |
− | + | newNumAudioStreams=None | |
− | + | ||
− | + | ||
− | + | def compressVideo(s): | |
− | + | ################################################################# | |
− | + | # choose partition to save compressed video | |
− | + | # a partition with enough free space is required | |
− | + | chosenDestination=None | |
− | + | for d in DESTINATION: | |
− | + | if enoughSpaceAvailable(d, space=freeSpaceRequired): | |
− | + | chosenDestination=d | |
− | + | break | |
− | + | if chosenDestination is None: | |
− | + | error(s, "Could not select a partition with enough free space") | |
− | + | else: | |
− | + | debug(s, "Chose following partition as destination for compressed video:\n\t" | |
− | + | + chosenDestination) | |
− | + | ||
− | + | TEMPDIR=s.tempDir | |
− | + | ################################################################# | |
− | + | # choose video file for compression | |
− | + | v=videoFile() | |
− | + | v.filename=selectVideoFile(SOURCES, minimumAge) | |
− | + | if v.filename is None: | |
− | + | error(s, "No videofile could be selected.") | |
− | + | v.basename=basename(v.filename) | |
− | + | debug(s, "Chose videofile: " + v.filename) | |
− | + | ||
− | + | try: | |
− | + | v.filesize=os.path.getsize(v.filename) | |
− | + | except OSError, e: | |
− | + | error(s, "Could not get mpg filesize.") | |
− | + | ||
− | + | # prefix for LOGGING | |
− | + | s.prefix=v.basename | |
− | + | tokens=v.basename.replace(".mpg","").split("_") | |
− | + | ||
− | + | if len(tokens) != 2: | |
− | + | error(s, "Could not parse filename for chanid and starttime.") | |
− | + | ||
− | + | try: | |
− | + | v.channelId=int(tokens[0]) | |
− | + | v.channelStart=tokens[1] | |
− | + | v.channelDate=datetime.strptime(v.channelStart, "%Y%m%d%H%M%S") | |
− | + | except ValueError: | |
− | + | error(s, "Could not parse channel id and date: " + "_".join(tokens)) | |
− | + | # get video information | |
− | + | v.videoInfo=getVideoStreamInformation(s, v.filename) | |
− | + | v.resolution=getVideoResolution(v.videoInfo) | |
− | + | v.duration=getVideoDuration(v.videoInfo) | |
− | + | v.numAudioStreams=getNumberOfAudioStreams(v.videoInfo) | |
− | + | ||
− | + | v.creation=getFileCreationDate(v.filename) | |
− | + | ||
− | + | v.newBasename=str(v.channelId) + "_" + str(v.channelStart) + ".avi" | |
− | + | v.newFilename=chosenDestination + v.newBasename | |
− | + | ||
− | + | v.tmpBasename=str(v.channelId) + "_" + str(v.channelStart) + ".tmp" | |
− | + | v.tmpFilename=TEMPDIR + "/" + v.newBasename | |
− | + | ||
− | + | ||
− | + | debug(s, str(v.channelId) + "," + str(v.channelStart) + "," + str(v.channelDate) + ":\n" | |
− | + | + "filename:\t" + v.filename + "\n" | |
− | + | + "resolution:\t" + 'x'.join(map(str,v.resolution)) + "\n" | |
− | + | ) | |
− | + | debug(s, "Temporary Directory: " + TEMPDIR) | |
− | + | debug(s, "Temporary filename : " + v.tmpFilename) | |
− | + | debug(s, "Temporary basename : " + v.tmpBasename ) | |
− | + | debug(s, "New basename : " + v.newBasename ) | |
− | + | debug(s, "New File will be : " + v.newFilename) | |
− | + | ||
− | + | ||
− | + | ################################################################# | |
− | + | # update mysql db | |
− | + | mysql = MySQLdb.connect(mysql_opts['host'], mysql_opts['user'], mysql_opts['pass'], mysql_opts['db']) | |
− | + | cursor = mysql.cursor() | |
− | + | sql="SELECT title FROM recorded WHERE chanid='%s' AND starttime='%s'" | |
− | + | sql=sql % (v.channelId , v.channelDate.strftime('%Y-%m-%d %H:%M:%S')) | |
− | + | debug(s, "Mysql database query for title") | |
− | + | debug(s, sql) | |
− | + | cursor.execute(sql) | |
− | + | #cursor.execute(sql, (v.channelId, v.channelDate.strftime('%Y-%m-%d %H:%M:%S'))) | |
− | + | videoTable=cursor.fetchall() | |
− | + | try: | |
− | + | v.title=videoTable[0][0] | |
− | + | if len(v.title) == 0: | |
− | + | error(s, "Title has zero length") | |
− | + | except IndexError, e: | |
− | + | error(s, "No mysql database entry found for this video.") | |
− | + | ||
− | + | debug(s, "Title of video is : " + v.title) | |
− | + | ||
− | + | # fix seeking and bookmarking | |
− | + | debug(s, "fix Seeking and bookmarking") | |
− | + | delete1="DELETE FROM `recordedseek` WHERE chanid='%s' AND starttime='%s'" | |
− | + | delete1=delete1 % (v.channelId, v.channelStart) | |
− | + | debug(s, delete1) | |
− | + | cursor.execute(delete1)#, (v.channelId, v.channelStart)) | |
− | + | ||
− | + | delete2="DELETE FROM `recordedmarkup` WHERE chanid='%s' AND starttime='%s'" | |
− | + | delete2=delete2 % (v.channelId, v.channelStart) | |
− | + | debug(s, delete2) | |
− | + | cursor.execute(delete2)#, (v.channelId, v.channelStart)) | |
− | + | ||
− | + | updateBase1="UPDATE `recorded` SET basename='%s' WHERE chanid='%s' AND starttime='%s';" | |
− | + | debug(s, "\nrenaming file in database") | |
− | + | updateBase1=updateBase1 % (v.tmpBasename, v.channelId, v.channelStart) | |
− | + | debug(s, updateBase1) | |
− | + | cursor.execute(updateBase1)#, (v.tmpBasename, v.channelId, v.channelStart)) | |
− | + | ||
− | + | ################################################################# | |
− | + | # set conversion parameter | |
− | + | ffCommand="" | |
− | + | ffOutput=" " + v.tmpFilename | |
− | + | # TODO remove ffClip | |
− | + | if v.resolution == (1280,720): | |
− | + | debug(s, "Compression profile for 1280") | |
− | + | ffDefault=" -c copy" | |
− | + | ffVideoCodec=" -c:v libx264 -crf:v 23 -preset:v slow -tune:v film" | |
− | + | ffAudioCodec=" -c:a libvorbis -q:a 5" | |
− | + | ffSize=" -s 960x540" | |
− | + | ffRate=" -r 25" | |
− | + | ffMaps=" -map 0:v -map 0:a"# -map 0:s" | |
− | + | ffClip=" -ss 16 -t 100" | |
− | + | ffClip="" | |
− | + | ffCommand=ffmpegBinary + " -i " + v.filename | |
− | + | ffCommand+=ffDefault + ffVideoCodec + ffAudioCodec + ffRate + ffSize + ffMaps + ffClip + ffOutput | |
− | + | elif v.resolution == (720,576): | |
− | + | debug(s, "Compression profile for720") | |
− | + | ffDefault=" -c copy" | |
− | + | ffVideoCodec=" -c:v libx264 -crf:v 23 -preset:v slow -tune:v film" | |
− | + | ffAudioCodec=" -c:a libvorbis -q:a 5" | |
− | + | #ffSize=" -s 960x540" | |
− | + | ffSize="" | |
− | + | ffMaps=" -map 0:v -map 0:a"# -map 0:s" | |
− | + | ffClip=" -ss 16 -t 100" | |
− | + | ffClip="" | |
− | + | ffCommand=ffmpegBinary + " -i " + v.filename | |
− | + | ffCommand+=ffDefault + ffVideoCodec + ffAudioCodec + ffSize + ffMaps + ffClip + ffOutput | |
− | + | else: | |
− | + | error(s, "unsupported format") | |
− | + | ||
− | + | ||
− | + | ################################################################# | |
− | + | # compress video file | |
− | + | cmd=shlex.split(ffCommand) | |
− | + | debug(s, "compressing video file...") | |
− | + | debug(s, "FFMPEG command:\n" + ffCommand) | |
− | + | p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) | |
− | + | stdout, stderr = p.communicate() | |
− | + | ||
− | + | print "Returncode:",p.returncode | |
− | + | if os.path.isfile(v.tmpFilename): | |
− | + | debug(s, "Video created.") | |
− | + | else: | |
− | + | print stdout | |
− | + | print stderr | |
− | + | error(s, "No video created.") | |
− | + | ||
− | + | ################################################################# | |
− | + | # check new filesize | |
− | + | try: | |
− | + | v.newFilesize=os.path.getsize(v.tmpFilename) | |
− | + | except OSError, e: | |
− | + | error(s, "Could not get new filesize.") | |
− | + | ||
− | + | ################################################################# | |
− | + | # compare video duration of result video | |
− | + | v.newVideoInfo=getVideoStreamInformation(s, v.tmpFilename) | |
− | + | v.newDuration=getVideoDuration(v.newVideoInfo) | |
− | + | v.newNumAudioStreams=getNumberOfAudioStreams(v.newVideoInfo) | |
− | + | difference=v.duration - v.newDuration | |
− | + | debug(s, "Duration difference : " + str(difference)) | |
− | + | if math.fabs(difference) > 2: | |
− | + | debug(s, "Duration of original video: " + str(v.duration)) | |
− | + | debug(s, "Duration of new video : " + str(v.newDuration)) | |
− | + | error(s, "Duration of result video is not equal.") | |
− | + | ||
− | + | ################################################################# | |
− | + | # all good ... | |
− | + | # ... move compressed video file to destination | |
− | + | debug(s, "moving " + v.tmpFilename + " to " + v.newFilename) | |
− | + | try: | |
− | + | shutil.move(v.tmpFilename,v.newFilename) | |
− | + | except shutil.Error, e: | |
− | + | print "Could not move compressed video file" | |
− | + | error(s, e) | |
− | + | ||
− | + | if os.path.isfile(v.newFilename): | |
− | + | debug(s, "Video successfully moved.") | |
− | + | else: | |
− | + | error(s, "Moved videofile does not exist.") | |
− | + | ################################################################# | |
− | + | # rename file in database | |
− | + | updateBase2="UPDATE recorded SET basename='%s', filesize='%s' WHERE chanid='%s' AND starttime='%s';" | |
− | + | updateBase2=updateBase2 % (v.newBasename, v.newFilesize, v.channelId, v.channelStart) | |
− | + | # TODO | |
− | + | # write transcoded="1" | |
− | + | # | |
− | + | debug(s, updateBase2) | |
− | + | cursor.execute(updateBase2) | |
− | + | ||
− | + | ################################################################# | |
− | + | # delete pngs | |
− | + | # | |
− | + | debug(s, "deleting... " + v.filename + "*png") | |
− | + | for filePath in glob.glob(v.filename + "*png"): | |
− | + | if os.path.isfile(filePath): | |
− | + | debug(s, "delete " + filePath) | |
− | + | os.remove(filePath) | |
− | + | ||
− | + | if os.path.isfile(v.filename): | |
− | + | debug(s, "deleting original videofile: " + v.filename) | |
− | + | os.remove(v.filename) | |
− | + | else: | |
− | + | error(s, "Could not delete original video") | |
− | + | ||
− | + | ################################################################# | |
− | + | clean(s) | |
− | + | #sys.exit(0) | |
− | + | def allowedTimes(): | |
− | + | now = datetime.now() | |
− | + | hour=int(datetime.strftime(now, "%H")) | |
− | + | if hour > 0 and not (hour > 17): | |
− | + | return True | |
− | + | else: | |
− | + | return False | |
− | + | ||
− | + | ||
− | + | s=status() | |
− | + | s.prefix="system: " | |
− | + | # cleaning up temporary directory | |
− | + | for filePath in glob.glob(TEMPPARENTDIR + "/*"): | |
− | + | if os.path.isdir(filePath): | |
− | + | debug(s, "delete old temporary directory " + filePath) | |
− | + | shutil.rmtree(filePath) | |
− | + | ||
− | + | count=0 | |
− | + | #while count < 1: | |
− | + | while True: | |
− | + | s=status() | |
− | + | s.prefix="system: " | |
− | + | TEMPDIR=createTemp(s, TEMPPARENTDIR) | |
− | + | s.tempDir=TEMPDIR | |
− | + | ||
− | + | if not enoughSpaceAvailable(TEMPDIR, space=freeSpaceInTempRequired): | |
− | + | error(s, "Not enough space free in temporary directory!!") | |
− | + | atexit.register(clean, status=s) | |
− | + | ||
− | + | ############################ | |
− | + | # video compression logic | |
− | + | if allowedTimes(): | |
− | + | count+=1 | |
− | + | debug(s, "Video number: " + str(count)) | |
+ | compressVideo(s) | ||
+ | |||
+ | time.sleep(600) | ||
+ | </pre> | ||
+ | }} | ||
+ | |||
+ | |||
+ | '''mythFunctions.py''' | ||
+ | {{python|mythFunctions.py| | ||
+ | <pre> | ||
+ | #!/usr/bin/env python | ||
+ | |||
+ | import json | ||
+ | import shlex, subprocess | ||
+ | import MySQLdb | ||
+ | import os, sys, glob | ||
+ | import tempfile | ||
+ | import time | ||
+ | from datetime import datetime, timedelta | ||
+ | from os.path import basename | ||
+ | import re | ||
+ | import shutil | ||
+ | |||
+ | |||
+ | PID=os.getpgid(0) | ||
+ | ffprobeBinary="/home/rfs/bin/ffprobe" | ||
+ | |||
+ | from mythLogging import error | ||
+ | |||
+ | ####################################################################################### | ||
+ | def getDisksize(filename): | ||
+ | df = subprocess.Popen(["/bin/df", filename], stdout=subprocess.PIPE) | ||
+ | output = df.communicate()[0] | ||
+ | print output | ||
+ | device, size, used, available, percent, mountpoint = output.split("\n")[1].split() | ||
+ | |||
+ | def get_fs_freespace(pathname): | ||
+ | "Get the free space of the filesystem containing pathname" | ||
+ | stat= os.statvfs(pathname) | ||
+ | # use f_bfree for superuser, or f_bavail if filesystem | ||
+ | # has reserved space for superuser | ||
+ | return int(stat.f_bfree*stat.f_bsize) | ||
+ | |||
+ | def enoughSpaceAvailable(path, space=20000000000): | ||
+ | return get_fs_freespace(path) > space | ||
+ | |||
+ | def getFileCreationDate(filename): | ||
+ | t = os.path.getmtime(filename) | ||
+ | then=datetime.fromtimestamp(t) | ||
+ | return then | ||
+ | |||
+ | def isMPGvideoPath(path): | ||
+ | return os.path.isfile(path) and path.endswith(".mpg") | ||
+ | |||
+ | def isMPGvideoFile(f, directory): | ||
+ | filename=directory + "/" + f | ||
+ | return os.path.isfile(filename) and f.endswith(".mpg") | ||
+ | |||
+ | # video suchen, das aelter als minimumAge ist | ||
+ | def selectVideoFile(SOURCES, minimumAge): | ||
+ | result=None | ||
+ | now = datetime.now() | ||
+ | for directory in SOURCES: | ||
+ | files = [f for f in os.listdir( directory ) if isMPGvideoFile(f, directory)] | ||
+ | for f in files: | ||
+ | filename=directory + "/" + f | ||
+ | then=getFileCreationDate(filename) | ||
+ | if (now - then) > minimumAge: | ||
+ | return filename | ||
+ | return result | ||
+ | |||
+ | def getVideoStreamInformation(status, pathtovideo): | ||
+ | stdout = "" | ||
+ | stderr = "" | ||
+ | try: | ||
+ | cmd=ffprobeBinary + " -print_format json -show_streams -loglevel quiet " | ||
+ | cmd+=pathtovideo | ||
+ | p = subprocess.Popen(shlex.split(cmd), | ||
+ | stdout=subprocess.PIPE, | ||
+ | stderr=subprocess.PIPE) | ||
+ | stdout, stderr = p.communicate() | ||
+ | except OSError: | ||
+ | error(status, "Could not get video duration with ffprobe.") | ||
+ | if len(stdout) == 0: | ||
+ | error(status, "No data read for file: " + pathtovideo) | ||
+ | |||
+ | data = json.loads(stdout) | ||
+ | streams=None | ||
+ | try: | ||
+ | streams=data['streams'] | ||
+ | except KeyError, e: | ||
+ | error(status, "Could not load stream information.") | ||
+ | return streams | ||
+ | |||
+ | def getVideoDuration(videoInfo): | ||
+ | duration=None | ||
+ | for s in videoInfo: | ||
+ | try: | ||
+ | codec_type=s['codec_type'] | ||
+ | if codec_type=="video": | ||
+ | d=s['duration'] | ||
+ | if len(d) > 0: | ||
+ | duration=float(d) | ||
+ | break | ||
+ | except KeyError, e: | ||
+ | pass | ||
+ | return duration | ||
+ | |||
+ | def getVideoResolution(videoInfo): | ||
+ | w=0; h=0 | ||
+ | for s in videoInfo: | ||
+ | try: | ||
+ | codec_type=s['codec_type'] | ||
+ | if codec_type=="video": | ||
+ | w=int(s['width']) | ||
+ | h=int(s['height']) | ||
+ | break | ||
+ | except KeyError, e: | ||
+ | pass | ||
+ | return (w,h) | ||
+ | |||
+ | def getNumberOfAudioStreams(videoInfo): | ||
+ | count=0 | ||
+ | for s in videoInfo: | ||
+ | try: | ||
+ | codec_type=s['codec_type'] | ||
+ | if codec_type=="audio": | ||
+ | count=count+1 | ||
+ | except KeyError, e: | ||
+ | pass | ||
+ | return count | ||
+ | |||
+ | </pre> | ||
+ | }} | ||
+ | |||
+ | '''mythLogging.py''' | ||
+ | {{python|mythLogging.py| | ||
+ | <pre> | ||
+ | #!/usr/bin/env python | ||
+ | import sys,os, shutil | ||
+ | |||
+ | PRINTDEBUG=True | ||
+ | |||
+ | def debug(status, message): | ||
+ | if PRINTDEBUG: | ||
+ | for line in message.split("\n"): | ||
+ | print "STATUS:" + status.prefix + " " + line | ||
+ | |||
+ | def error(status, message): | ||
+ | for line in message.split("\n"): | ||
+ | sys.stderr.write("ERROR: " + status.prefix + " " + str(line) + "\n") | ||
+ | print "ERROR: " + status.prefix + " " + str(line) | ||
+ | clean(status) | ||
+ | sys.exit(1) | ||
+ | |||
+ | def clean(status): | ||
+ | os.chdir(os.path.expanduser("~")) | ||
+ | tempDir=status.tempDir | ||
+ | if not tempDir is None: | ||
+ | # remove temporary directory | ||
+ | if os.path.exists(tempDir): | ||
+ | debug(status, "Removing temporary directory " + tempDir + ".") | ||
+ | shutil.rmtree(tempDir) | ||
+ | |||
+ | </pre> | ||
+ | }} | ||
+ | |||
+ | {{Code_box|compressVideos| | ||
+ | <pre> | ||
+ | #! /bin/sh | ||
+ | # | ||
+ | # /etc/init.d/vncserver this Script | ||
+ | # /usr/bin/vncserver Program | ||
+ | # | ||
+ | |||
+ | # Check for missing binaries | ||
+ | AUTOSSH_BIN=/home/rfs/bin/mythCompress.py | ||
+ | test -x $AUTOSSH_BIN || exit 5 | ||
+ | |||
+ | LOG="/home/rfs/logs/mythCompress.syslog" | ||
+ | |||
+ | case "$1" in | ||
+ | start) | ||
+ | echo "Starting automatic video compression" | ||
+ | sudo -u rfs /home/rfs/bin/mythCompress.py >> $LOG 2>> $LOG & | ||
+ | echo "Started" | ||
+ | ;; | ||
+ | # | ||
+ | stop) | ||
+ | echo "Stopping automatic video compression not implemented" | ||
+ | echo 'ps axu | grep "mythCompress" | grep -v "init.d" | grep -v grep | awk '{print $2}' | xargs kill' | ||
+ | ;; | ||
+ | # | ||
+ | # | ||
+ | *) | ||
+ | echo "Usage: $0 {start|stop}" | ||
+ | exit 1 | ||
+ | ;; | ||
+ | esac | ||
+ | |||
+ | </pre> | ||
+ | }} |
Latest revision as of 19:23, 21 October 2013
Due to disc restrictions in my mythbox i decided to write a script to compress all my videos. Mythtranscode does this but the videos have been recorded quite some time ago and are spread over multiple discs. Compressing those videos to a single destination is no option because they probably won't fit onto one partition. Therefore i wrote a script that dynamically picks a show that is at least two weeks old, compresses it and saves it to a partition with enough disc space. So, there are multiple folders where videos are waiting for compression and multiple folders where compressed videos are stored. Actually, these folders can be the same, but they don't have to be as long they are visible to mythtv (see mythtv-setup and storing directories). For compressed videos i created another storing group called 'compressed' and added all destination folders. Videos are recorded in two different formats by my tv-card (PAL and 720p). This script decides upon video resolution which parameters to pick. Additionally i wanted this script to run as a system service and between 1 am and 6 pm.
Processing:
- Is it in the right time slot? (1am until 6pm)
- Pick a video for compression
- Temporarily rename video in mysql database
- Compress video in temporary folder
- Check if compressed video is okay (duration check)
- Update mysql database
- Back to 1.
Following programs are required: python 2.7, ffmpeg (ffmpeg version git-2013-10-01-2e2a2d8), ffprobe (comes with ffmpeg)
This script consists of three python scripts: mythCompress.py, mythLogging.py, mythFunctions.py and one system (init.d) script: compressVideos
mythCompress.py
#!/usr/bin/env python import MySQLdb import os, sys, glob import shlex, subprocess import tempfile import time from datetime import datetime, timedelta from os.path import basename import re import shutil import math import atexit from mythFunctions import * from mythLogging import error, debug, clean import psutil os.nice(19) program = psutil.Process(os.getpid()) program.set_ionice(psutil.IOPRIO_CLASS_IDLE) ###################################################################### # SETTINGS # # folders to save compressed videos DESTINATION=[ "/media/myth/compressed/" , "/media/data/mythtv/compressed/" ] # folders to read uncompressed videos SOURCES=[ "/media/data/mythtv/default/" , "/media/myth/default/" ] TEMPPARENTDIR=os.path.expanduser("~/mythtv-tmp/") PRINTDEBUG=True mysql_opts = { 'host': "localhost", 'user': "mythtv", 'pass': "CHANGEME", 'db': "mythconverg" } minimumAge = timedelta(days=14) # for conversion ffmpegBinary="/home/rfs/bin/ffmpeg" ffprobeBinary="/home/rfs/bin/ffprobe" freeSpaceRequired=20000000000 freeSpaceInTempRequired=10000000000 ###################################################################### # constants PID=os.getpgid(0) now=datetime.now() ################################################################# # functions def createTemp(status, parentDirectory): tempDir=None if os.path.exists(parentDirectory): tempDir = tempfile.mkdtemp(prefix=parentDirectory + str(PID) + "-") debug(status, "Temporary directory: " + tempDir) else: status.tempDir=None error(status, "Parent directory for temp does not exist.") # change directory to working directory try: os.chdir(tempDir) except OSError: status.tempDir=None error(status, "Could not find temporary directory (OSError).") # return result return tempDir class status(object): prefix=None tempDir=None class videoFile(object): channelId=None channelStart=None channelDate=None title=None # quellvideo filename=None basename=None videoInfo=None duration=None numAudioStreams=None # temporaere ausgabe tmpFilename=None tmpBasename=None # ziel newFilename=None newBasename=None newVideoInfo=None newDuration=None newNumAudioStreams=None def compressVideo(s): ################################################################# # choose partition to save compressed video # a partition with enough free space is required chosenDestination=None for d in DESTINATION: if enoughSpaceAvailable(d, space=freeSpaceRequired): chosenDestination=d break if chosenDestination is None: error(s, "Could not select a partition with enough free space") else: debug(s, "Chose following partition as destination for compressed video:\n\t" + chosenDestination) TEMPDIR=s.tempDir ################################################################# # choose video file for compression v=videoFile() v.filename=selectVideoFile(SOURCES, minimumAge) if v.filename is None: error(s, "No videofile could be selected.") v.basename=basename(v.filename) debug(s, "Chose videofile: " + v.filename) try: v.filesize=os.path.getsize(v.filename) except OSError, e: error(s, "Could not get mpg filesize.") # prefix for LOGGING s.prefix=v.basename tokens=v.basename.replace(".mpg","").split("_") if len(tokens) != 2: error(s, "Could not parse filename for chanid and starttime.") try: v.channelId=int(tokens[0]) v.channelStart=tokens[1] v.channelDate=datetime.strptime(v.channelStart, "%Y%m%d%H%M%S") except ValueError: error(s, "Could not parse channel id and date: " + "_".join(tokens)) # get video information v.videoInfo=getVideoStreamInformation(s, v.filename) v.resolution=getVideoResolution(v.videoInfo) v.duration=getVideoDuration(v.videoInfo) v.numAudioStreams=getNumberOfAudioStreams(v.videoInfo) v.creation=getFileCreationDate(v.filename) v.newBasename=str(v.channelId) + "_" + str(v.channelStart) + ".avi" v.newFilename=chosenDestination + v.newBasename v.tmpBasename=str(v.channelId) + "_" + str(v.channelStart) + ".tmp" v.tmpFilename=TEMPDIR + "/" + v.newBasename debug(s, str(v.channelId) + "," + str(v.channelStart) + "," + str(v.channelDate) + ":\n" + "filename:\t" + v.filename + "\n" + "resolution:\t" + 'x'.join(map(str,v.resolution)) + "\n" ) debug(s, "Temporary Directory: " + TEMPDIR) debug(s, "Temporary filename : " + v.tmpFilename) debug(s, "Temporary basename : " + v.tmpBasename ) debug(s, "New basename : " + v.newBasename ) debug(s, "New File will be : " + v.newFilename) ################################################################# # update mysql db mysql = MySQLdb.connect(mysql_opts['host'], mysql_opts['user'], mysql_opts['pass'], mysql_opts['db']) cursor = mysql.cursor() sql="SELECT title FROM recorded WHERE chanid='%s' AND starttime='%s'" sql=sql % (v.channelId , v.channelDate.strftime('%Y-%m-%d %H:%M:%S')) debug(s, "Mysql database query for title") debug(s, sql) cursor.execute(sql) #cursor.execute(sql, (v.channelId, v.channelDate.strftime('%Y-%m-%d %H:%M:%S'))) videoTable=cursor.fetchall() try: v.title=videoTable[0][0] if len(v.title) == 0: error(s, "Title has zero length") except IndexError, e: error(s, "No mysql database entry found for this video.") debug(s, "Title of video is : " + v.title) # fix seeking and bookmarking debug(s, "fix Seeking and bookmarking") delete1="DELETE FROM `recordedseek` WHERE chanid='%s' AND starttime='%s'" delete1=delete1 % (v.channelId, v.channelStart) debug(s, delete1) cursor.execute(delete1)#, (v.channelId, v.channelStart)) delete2="DELETE FROM `recordedmarkup` WHERE chanid='%s' AND starttime='%s'" delete2=delete2 % (v.channelId, v.channelStart) debug(s, delete2) cursor.execute(delete2)#, (v.channelId, v.channelStart)) updateBase1="UPDATE `recorded` SET basename='%s' WHERE chanid='%s' AND starttime='%s';" debug(s, "\nrenaming file in database") updateBase1=updateBase1 % (v.tmpBasename, v.channelId, v.channelStart) debug(s, updateBase1) cursor.execute(updateBase1)#, (v.tmpBasename, v.channelId, v.channelStart)) ################################################################# # set conversion parameter ffCommand="" ffOutput=" " + v.tmpFilename # TODO remove ffClip if v.resolution == (1280,720): debug(s, "Compression profile for 1280") ffDefault=" -c copy" ffVideoCodec=" -c:v libx264 -crf:v 23 -preset:v slow -tune:v film" ffAudioCodec=" -c:a libvorbis -q:a 5" ffSize=" -s 960x540" ffRate=" -r 25" ffMaps=" -map 0:v -map 0:a"# -map 0:s" ffClip=" -ss 16 -t 100" ffClip="" ffCommand=ffmpegBinary + " -i " + v.filename ffCommand+=ffDefault + ffVideoCodec + ffAudioCodec + ffRate + ffSize + ffMaps + ffClip + ffOutput elif v.resolution == (720,576): debug(s, "Compression profile for720") ffDefault=" -c copy" ffVideoCodec=" -c:v libx264 -crf:v 23 -preset:v slow -tune:v film" ffAudioCodec=" -c:a libvorbis -q:a 5" #ffSize=" -s 960x540" ffSize="" ffMaps=" -map 0:v -map 0:a"# -map 0:s" ffClip=" -ss 16 -t 100" ffClip="" ffCommand=ffmpegBinary + " -i " + v.filename ffCommand+=ffDefault + ffVideoCodec + ffAudioCodec + ffSize + ffMaps + ffClip + ffOutput else: error(s, "unsupported format") ################################################################# # compress video file cmd=shlex.split(ffCommand) debug(s, "compressing video file...") debug(s, "FFMPEG command:\n" + ffCommand) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() print "Returncode:",p.returncode if os.path.isfile(v.tmpFilename): debug(s, "Video created.") else: print stdout print stderr error(s, "No video created.") ################################################################# # check new filesize try: v.newFilesize=os.path.getsize(v.tmpFilename) except OSError, e: error(s, "Could not get new filesize.") ################################################################# # compare video duration of result video v.newVideoInfo=getVideoStreamInformation(s, v.tmpFilename) v.newDuration=getVideoDuration(v.newVideoInfo) v.newNumAudioStreams=getNumberOfAudioStreams(v.newVideoInfo) difference=v.duration - v.newDuration debug(s, "Duration difference : " + str(difference)) if math.fabs(difference) > 2: debug(s, "Duration of original video: " + str(v.duration)) debug(s, "Duration of new video : " + str(v.newDuration)) error(s, "Duration of result video is not equal.") ################################################################# # all good ... # ... move compressed video file to destination debug(s, "moving " + v.tmpFilename + " to " + v.newFilename) try: shutil.move(v.tmpFilename,v.newFilename) except shutil.Error, e: print "Could not move compressed video file" error(s, e) if os.path.isfile(v.newFilename): debug(s, "Video successfully moved.") else: error(s, "Moved videofile does not exist.") ################################################################# # rename file in database updateBase2="UPDATE recorded SET basename='%s', filesize='%s' WHERE chanid='%s' AND starttime='%s';" updateBase2=updateBase2 % (v.newBasename, v.newFilesize, v.channelId, v.channelStart) # TODO # write transcoded="1" # debug(s, updateBase2) cursor.execute(updateBase2) ################################################################# # delete pngs # debug(s, "deleting... " + v.filename + "*png") for filePath in glob.glob(v.filename + "*png"): if os.path.isfile(filePath): debug(s, "delete " + filePath) os.remove(filePath) if os.path.isfile(v.filename): debug(s, "deleting original videofile: " + v.filename) os.remove(v.filename) else: error(s, "Could not delete original video") ################################################################# clean(s) #sys.exit(0) def allowedTimes(): now = datetime.now() hour=int(datetime.strftime(now, "%H")) if hour > 0 and not (hour > 17): return True else: return False s=status() s.prefix="system: " # cleaning up temporary directory for filePath in glob.glob(TEMPPARENTDIR + "/*"): if os.path.isdir(filePath): debug(s, "delete old temporary directory " + filePath) shutil.rmtree(filePath) count=0 #while count < 1: while True: s=status() s.prefix="system: " TEMPDIR=createTemp(s, TEMPPARENTDIR) s.tempDir=TEMPDIR if not enoughSpaceAvailable(TEMPDIR, space=freeSpaceInTempRequired): error(s, "Not enough space free in temporary directory!!") atexit.register(clean, status=s) ############################ # video compression logic if allowedTimes(): count+=1 debug(s, "Video number: " + str(count)) compressVideo(s) time.sleep(600)
mythFunctions.py
#!/usr/bin/env python import json import shlex, subprocess import MySQLdb import os, sys, glob import tempfile import time from datetime import datetime, timedelta from os.path import basename import re import shutil PID=os.getpgid(0) ffprobeBinary="/home/rfs/bin/ffprobe" from mythLogging import error ####################################################################################### def getDisksize(filename): df = subprocess.Popen(["/bin/df", filename], stdout=subprocess.PIPE) output = df.communicate()[0] print output device, size, used, available, percent, mountpoint = output.split("\n")[1].split() def get_fs_freespace(pathname): "Get the free space of the filesystem containing pathname" stat= os.statvfs(pathname) # use f_bfree for superuser, or f_bavail if filesystem # has reserved space for superuser return int(stat.f_bfree*stat.f_bsize) def enoughSpaceAvailable(path, space=20000000000): return get_fs_freespace(path) > space def getFileCreationDate(filename): t = os.path.getmtime(filename) then=datetime.fromtimestamp(t) return then def isMPGvideoPath(path): return os.path.isfile(path) and path.endswith(".mpg") def isMPGvideoFile(f, directory): filename=directory + "/" + f return os.path.isfile(filename) and f.endswith(".mpg") # video suchen, das aelter als minimumAge ist def selectVideoFile(SOURCES, minimumAge): result=None now = datetime.now() for directory in SOURCES: files = [f for f in os.listdir( directory ) if isMPGvideoFile(f, directory)] for f in files: filename=directory + "/" + f then=getFileCreationDate(filename) if (now - then) > minimumAge: return filename return result def getVideoStreamInformation(status, pathtovideo): stdout = "" stderr = "" try: cmd=ffprobeBinary + " -print_format json -show_streams -loglevel quiet " cmd+=pathtovideo p = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() except OSError: error(status, "Could not get video duration with ffprobe.") if len(stdout) == 0: error(status, "No data read for file: " + pathtovideo) data = json.loads(stdout) streams=None try: streams=data['streams'] except KeyError, e: error(status, "Could not load stream information.") return streams def getVideoDuration(videoInfo): duration=None for s in videoInfo: try: codec_type=s['codec_type'] if codec_type=="video": d=s['duration'] if len(d) > 0: duration=float(d) break except KeyError, e: pass return duration def getVideoResolution(videoInfo): w=0; h=0 for s in videoInfo: try: codec_type=s['codec_type'] if codec_type=="video": w=int(s['width']) h=int(s['height']) break except KeyError, e: pass return (w,h) def getNumberOfAudioStreams(videoInfo): count=0 for s in videoInfo: try: codec_type=s['codec_type'] if codec_type=="audio": count=count+1 except KeyError, e: pass return count
mythLogging.py
#!/usr/bin/env python import sys,os, shutil PRINTDEBUG=True def debug(status, message): if PRINTDEBUG: for line in message.split("\n"): print "STATUS:" + status.prefix + " " + line def error(status, message): for line in message.split("\n"): sys.stderr.write("ERROR: " + status.prefix + " " + str(line) + "\n") print "ERROR: " + status.prefix + " " + str(line) clean(status) sys.exit(1) def clean(status): os.chdir(os.path.expanduser("~")) tempDir=status.tempDir if not tempDir is None: # remove temporary directory if os.path.exists(tempDir): debug(status, "Removing temporary directory " + tempDir + ".") shutil.rmtree(tempDir)
#! /bin/sh # # /etc/init.d/vncserver this Script # /usr/bin/vncserver Program # # Check for missing binaries AUTOSSH_BIN=/home/rfs/bin/mythCompress.py test -x $AUTOSSH_BIN || exit 5 LOG="/home/rfs/logs/mythCompress.syslog" case "$1" in start) echo "Starting automatic video compression" sudo -u rfs /home/rfs/bin/mythCompress.py >> $LOG 2>> $LOG & echo "Started" ;; # stop) echo "Stopping automatic video compression not implemented" echo 'ps axu | grep "mythCompress" | grep -v "init.d" | grep -v grep | awk '{print $2}' | xargs kill' ;; # # *) echo "Usage: $0 {start|stop}" exit 1 ;; esac