Difference between revisions of "MythCompress"

From MythTV Official Wiki
Jump to: navigation, search
m (wiki formatting)
m (more wiki formatting)
Line 16: Line 16:
 
and one system script: compressVideos
 
and one system script: compressVideos
  
−
#!/usr/bin/env python
+
{{python|compressVideos.py|
−
+
<pre>
−
import MySQLdb
+
#!/usr/bin/env python
−
import os, sys, glob
+
 
−
import shlex, subprocess
+
import MySQLdb
−
import tempfile
+
import os, sys, glob
−
import time
+
import shlex, subprocess
−
from datetime import datetime, timedelta
+
import tempfile
−
from os.path import basename
+
import time
−
import re
+
from datetime import datetime, timedelta
−
import shutil
+
from os.path import basename
−
import math
+
import re
−
import atexit
+
import shutil
−
+
import math
−
from mythFunctions import *
+
import atexit
−
from mythLogging import error, debug, clean
+
 
−
+
from mythFunctions import *
−
import psutil
+
from mythLogging import error, debug, clean
−
os.nice(19)
+
 
−
program = psutil.Process(os.getpid())
+
import psutil
−
program.set_ionice(psutil.IOPRIO_CLASS_IDLE)
+
os.nice(19)
−
+
program = psutil.Process(os.getpid())
−
######################################################################
+
program.set_ionice(psutil.IOPRIO_CLASS_IDLE)
−
# SETTINGS
+
 
−
#
+
######################################################################
−
# folders to save compressed videos
+
# SETTINGS
−
DESTINATION=[ "/media/myth/compressed/" , "/media/data/mythtv/compressed/" ]
+
#
−
# folders to read uncompressed videos
+
# folders to save compressed videos
−
SOURCES=[ "/media/data/mythtv/default/" , "/media/myth/default/" ]  
+
DESTINATION=[ "/media/myth/compressed/" , "/media/data/mythtv/compressed/" ]
−
TEMPPARENTDIR=os.path.expanduser("~/mythtv-tmp/")
+
# folders to read uncompressed videos
−
PRINTDEBUG=True
+
SOURCES=[ "/media/data/mythtv/default/" , "/media/myth/default/" ]  
−
mysql_opts = {
+
TEMPPARENTDIR=os.path.expanduser("~/mythtv-tmp/")
−
    'host': "localhost",
+
PRINTDEBUG=True
−
    'user': "mythtv",
+
mysql_opts = {
−
    'pass': "CHANGEME",
+
    'host': "localhost",
−
    'db':  "mythconverg"
+
    'user': "mythtv",
−
    }  
+
    'pass': "CHANGEME",
−
minimumAge = timedelta(days=14) # for conversion
+
    'db':  "mythconverg"
−
ffmpegBinary="/home/rfs/bin/ffmpeg"
+
    }  
−
ffprobeBinary="/home/rfs/bin/ffprobe"
+
minimumAge = timedelta(days=14) # for conversion
−
+
ffmpegBinary="/home/rfs/bin/ffmpeg"
−
freeSpaceRequired=20000000000
+
ffprobeBinary="/home/rfs/bin/ffprobe"
−
freeSpaceInTempRequired=10000000000
+
 
−
######################################################################
+
freeSpaceRequired=20000000000
−
# constants
+
freeSpaceInTempRequired=10000000000
−
PID=os.getpgid(0)
+
######################################################################
−
now=datetime.now()
+
# constants
−
+
PID=os.getpgid(0)
−
#################################################################
+
now=datetime.now()
−
# functions
+
 
−
def createTemp(status, parentDirectory):
+
#################################################################
−
    tempDir=None
+
# functions
−
    if os.path.exists(parentDirectory):
+
def createTemp(status, parentDirectory):
−
        tempDir = tempfile.mkdtemp(prefix=parentDirectory + str(PID) + "-")
+
    tempDir=None
−
        debug(status, "Temporary directory: " + tempDir)
+
    if os.path.exists(parentDirectory):
−
    else:
+
        tempDir = tempfile.mkdtemp(prefix=parentDirectory + str(PID) + "-")
−
        status.tempDir=None
+
        debug(status, "Temporary directory: " + tempDir)
−
        error(status, "Parent directory for temp does not exist.")
+
    else:
−
    # change directory to working directory
+
        status.tempDir=None
−
    try:
+
        error(status, "Parent directory for temp does not exist.")
−
        os.chdir(tempDir)
+
    # change directory to working directory
−
    except OSError:
+
    try:
−
        status.tempDir=None
+
        os.chdir(tempDir)
−
        error(status, "Could not find temporary directory (OSError).")
+
    except OSError:
−
    # return result
+
        status.tempDir=None
−
    return tempDir
+
        error(status, "Could not find temporary directory (OSError).")
−
+
    # return result
−
class status(object):
+
    return tempDir
−
    prefix=None
+
 
−
    tempDir=None
+
class status(object):
−
+
    prefix=None
−
class videoFile(object):
+
    tempDir=None
−
    channelId=None
+
 
−
    channelStart=None
+
class videoFile(object):
−
    channelDate=None
+
    channelId=None
−
    title=None
+
    channelStart=None
−
+
    channelDate=None
−
    # quellvideo
+
    title=None
−
    filename=None
+
 
−
    basename=None
+
    # quellvideo
−
    videoInfo=None
+
    filename=None
−
    duration=None
+
    basename=None
−
    numAudioStreams=None
+
    videoInfo=None
−
+
    duration=None
−
    # temporaere ausgabe
+
    numAudioStreams=None
−
    tmpFilename=None
+
 
−
    tmpBasename=None
+
    # temporaere ausgabe
−
+
    tmpFilename=None
−
    # ziel
+
    tmpBasename=None
−
    newFilename=None
+
 
−
    newBasename=None
+
    # ziel
−
    newVideoInfo=None
+
    newFilename=None
−
    newDuration=None
+
    newBasename=None
−
    newNumAudioStreams=None
+
    newVideoInfo=None
−
   
+
    newDuration=None
−
+
    newNumAudioStreams=None
−
def compressVideo(s):
+
   
−
    #################################################################
+
 
−
    # choose partition to save compressed video
+
def compressVideo(s):
−
    # a partition with enough free space is required
+
    #################################################################
−
    chosenDestination=None
+
    # choose partition to save compressed video
−
    for d in DESTINATION:
+
    # a partition with enough free space is required
−
        if enoughSpaceAvailable(d, space=freeSpaceRequired):
+
    chosenDestination=None
−
            chosenDestination=d
+
    for d in DESTINATION:
−
            break
+
        if enoughSpaceAvailable(d, space=freeSpaceRequired):
−
    if chosenDestination is None:
+
            chosenDestination=d
−
        error(s, "Could not select a partition with enough free space")
+
            break
−
    else:
+
    if chosenDestination is None:
−
        debug(s, "Chose following partition as destination for compressed video:\n\t"  
+
        error(s, "Could not select a partition with enough free space")
−
              + chosenDestination)
+
    else:
−
+
        debug(s, "Chose following partition as destination for compressed video:\n\t"  
−
    TEMPDIR=s.tempDir
+
              + chosenDestination)
−
    #################################################################
+
 
−
    # choose video file for compression
+
    TEMPDIR=s.tempDir
−
    v=videoFile()
+
    #################################################################
−
    v.filename=selectVideoFile(SOURCES, minimumAge)
+
    # choose video file for compression
−
    if v.filename is None:
+
    v=videoFile()
−
        error(s, "No videofile could be selected.")
+
    v.filename=selectVideoFile(SOURCES, minimumAge)
−
    v.basename=basename(v.filename)
+
    if v.filename is None:
−
    debug(s, "Chose videofile: " + v.filename)
+
        error(s, "No videofile could be selected.")
−
+
    v.basename=basename(v.filename)
−
    try:
+
    debug(s, "Chose videofile: " + v.filename)
−
        v.filesize=os.path.getsize(v.filename)
+
 
−
    except OSError, e:
+
    try:
−
        error(s, "Could not get mpg filesize.")
+
        v.filesize=os.path.getsize(v.filename)
−
   
+
    except OSError, e:
−
    # prefix for LOGGING
+
        error(s, "Could not get mpg filesize.")
−
    s.prefix=v.basename
+
   
−
    tokens=v.basename.replace(".mpg","").split("_")
+
    # prefix for LOGGING
−
+
    s.prefix=v.basename
−
    if len(tokens) != 2:
+
    tokens=v.basename.replace(".mpg","").split("_")
−
        error(s, "Could not parse filename for chanid and starttime.")
+
 
−
+
    if len(tokens) != 2:
−
    try:
+
        error(s, "Could not parse filename for chanid and starttime.")
−
        v.channelId=int(tokens[0])
+
 
−
        v.channelStart=tokens[1]
+
    try:
−
        v.channelDate=datetime.strptime(v.channelStart, "%Y%m%d%H%M%S")
+
        v.channelId=int(tokens[0])
−
    except ValueError:
+
        v.channelStart=tokens[1]
−
        error(s, "Could not parse channel id and date: " + "_".join(tokens))
+
        v.channelDate=datetime.strptime(v.channelStart, "%Y%m%d%H%M%S")
−
    # get video information
+
    except ValueError:
−
    v.videoInfo=getVideoStreamInformation(s, v.filename)
+
        error(s, "Could not parse channel id and date: " + "_".join(tokens))
−
    v.resolution=getVideoResolution(v.videoInfo)
+
    # get video information
−
    v.duration=getVideoDuration(v.videoInfo)
+
    v.videoInfo=getVideoStreamInformation(s, v.filename)
−
    v.numAudioStreams=getNumberOfAudioStreams(v.videoInfo)
+
    v.resolution=getVideoResolution(v.videoInfo)
−
   
+
    v.duration=getVideoDuration(v.videoInfo)
−
    v.creation=getFileCreationDate(v.filename)
+
    v.numAudioStreams=getNumberOfAudioStreams(v.videoInfo)
−
+
   
−
    v.newBasename=str(v.channelId) + "_" + str(v.channelStart) + ".avi"
+
    v.creation=getFileCreationDate(v.filename)
−
    v.newFilename=chosenDestination + v.newBasename
+
 
−
+
    v.newBasename=str(v.channelId) + "_" + str(v.channelStart) + ".avi"
−
    v.tmpBasename=str(v.channelId) + "_" + str(v.channelStart) + ".tmp"
+
    v.newFilename=chosenDestination + v.newBasename
−
    v.tmpFilename=TEMPDIR + "/" + v.newBasename
+
 
−
+
    v.tmpBasename=str(v.channelId) + "_" + str(v.channelStart) + ".tmp"
−
+
    v.tmpFilename=TEMPDIR + "/" + v.newBasename
−
    debug(s, str(v.channelId)  + "," + str(v.channelStart) + "," + str(v.channelDate) + ":\n"
+
 
−
          + "filename:\t" + v.filename + "\n"
+
 
−
          + "resolution:\t" + 'x'.join(map(str,v.resolution)) + "\n"
+
    debug(s, str(v.channelId)  + "," + str(v.channelStart) + "," + str(v.channelDate) + ":\n"
−
          )
+
          + "filename:\t" + v.filename + "\n"
−
    debug(s, "Temporary Directory: " + TEMPDIR)
+
          + "resolution:\t" + 'x'.join(map(str,v.resolution)) + "\n"
−
    debug(s, "Temporary filename : " + v.tmpFilename)
+
          )
−
    debug(s, "Temporary basename : " + v.tmpBasename )
+
    debug(s, "Temporary Directory: " + TEMPDIR)
−
    debug(s, "New basename      : " + v.newBasename )
+
    debug(s, "Temporary filename : " + v.tmpFilename)
−
    debug(s, "New File will be  : " + v.newFilename)
+
    debug(s, "Temporary basename : " + v.tmpBasename )
−
+
    debug(s, "New basename      : " + v.newBasename )
−
+
    debug(s, "New File will be  : " + v.newFilename)
−
    #################################################################
+
 
−
    # update mysql db
+
 
−
    mysql = MySQLdb.connect(mysql_opts['host'], mysql_opts['user'], mysql_opts['pass'], mysql_opts['db'])  
+
    #################################################################
−
    cursor = mysql.cursor()
+
    # update mysql db
−
    sql="SELECT title FROM recorded WHERE chanid='%s' AND starttime='%s'"
+
    mysql = MySQLdb.connect(mysql_opts['host'], mysql_opts['user'], mysql_opts['pass'], mysql_opts['db'])  
−
    sql=sql % (v.channelId , v.channelDate.strftime('%Y-%m-%d %H:%M:%S'))
+
    cursor = mysql.cursor()
−
    debug(s, "Mysql database query for title")
+
    sql="SELECT title FROM recorded WHERE chanid='%s' AND starttime='%s'"
−
    debug(s, sql)
+
    sql=sql % (v.channelId , v.channelDate.strftime('%Y-%m-%d %H:%M:%S'))
−
    cursor.execute(sql)
+
    debug(s, "Mysql database query for title")
−
    #cursor.execute(sql, (v.channelId,  v.channelDate.strftime('%Y-%m-%d %H:%M:%S')))
+
    debug(s, sql)
−
    videoTable=cursor.fetchall()
+
    cursor.execute(sql)
−
    try:
+
    #cursor.execute(sql, (v.channelId,  v.channelDate.strftime('%Y-%m-%d %H:%M:%S')))
−
        v.title=videoTable[0][0]
+
    videoTable=cursor.fetchall()
−
        if len(v.title) == 0:
+
    try:
−
            error(s, "Title has zero length")
+
        v.title=videoTable[0][0]
−
    except IndexError, e:
+
        if len(v.title) == 0:
−
        error(s, "No mysql database entry found for this video.")
+
            error(s, "Title has zero length")
−
+
    except IndexError, e:
−
    debug(s, "Title of video is  : " + v.title)
+
        error(s, "No mysql database entry found for this video.")
−
+
 
−
    # fix seeking and bookmarking
+
    debug(s, "Title of video is  : " + v.title)
−
    debug(s, "fix Seeking and bookmarking")
+
 
−
    delete1="DELETE FROM `recordedseek` WHERE chanid='%s' AND starttime='%s'"
+
    # fix seeking and bookmarking
−
    delete1=delete1 % (v.channelId, v.channelStart)
+
    debug(s, "fix Seeking and bookmarking")
−
    debug(s, delete1)
+
    delete1="DELETE FROM `recordedseek` WHERE chanid='%s' AND starttime='%s'"
−
    cursor.execute(delete1)#, (v.channelId, v.channelStart))
+
    delete1=delete1 % (v.channelId, v.channelStart)
−
+
    debug(s, delete1)
−
    delete2="DELETE FROM `recordedmarkup` WHERE chanid='%s' AND starttime='%s'"
+
    cursor.execute(delete1)#, (v.channelId, v.channelStart))
−
    delete2=delete2 % (v.channelId, v.channelStart)
+
 
−
    debug(s, delete2)
+
    delete2="DELETE FROM `recordedmarkup` WHERE chanid='%s' AND starttime='%s'"
−
    cursor.execute(delete2)#, (v.channelId, v.channelStart))
+
    delete2=delete2 % (v.channelId, v.channelStart)
−
+
    debug(s, delete2)
−
    updateBase1="UPDATE `recorded` SET basename='%s' WHERE chanid='%s' AND starttime='%s';"
+
    cursor.execute(delete2)#, (v.channelId, v.channelStart))
−
    debug(s, "\nrenaming file in database")
+
 
−
    updateBase1=updateBase1 % (v.tmpBasename, v.channelId, v.channelStart)
+
    updateBase1="UPDATE `recorded` SET basename='%s' WHERE chanid='%s' AND starttime='%s';"
−
    debug(s, updateBase1)
+
    debug(s, "\nrenaming file in database")
−
    cursor.execute(updateBase1)#, (v.tmpBasename, v.channelId, v.channelStart))
+
    updateBase1=updateBase1 % (v.tmpBasename, v.channelId, v.channelStart)
−
+
    debug(s, updateBase1)
−
    #################################################################
+
    cursor.execute(updateBase1)#, (v.tmpBasename, v.channelId, v.channelStart))
−
    # set conversion parameter
+
 
−
    ffCommand=""
+
    #################################################################
−
    ffOutput=" " + v.tmpFilename
+
    # set conversion parameter
−
    # TODO remove ffClip
+
    ffCommand=""
−
    if v.resolution == (1280,720):
+
    ffOutput=" " + v.tmpFilename
−
        debug(s, "Compression profile for 1280")
+
    # TODO remove ffClip
−
        ffDefault=" -c copy"
+
    if v.resolution == (1280,720):
−
        ffVideoCodec=" -c:v libx264 -crf:v 23 -preset:v slow -tune:v film"
+
        debug(s, "Compression profile for 1280")
−
        ffAudioCodec=" -c:a libvorbis -q:a 5"
+
        ffDefault=" -c copy"
−
        ffSize=" -s 960x540"
+
        ffVideoCodec=" -c:v libx264 -crf:v 23 -preset:v slow -tune:v film"
−
        ffRate=" -r 25"
+
        ffAudioCodec=" -c:a libvorbis -q:a 5"
−
        ffMaps=" -map 0:v -map 0:a"# -map 0:s"
+
        ffSize=" -s 960x540"
−
        ffClip=" -ss 16 -t 100"
+
        ffRate=" -r 25"
−
        ffClip=""
+
        ffMaps=" -map 0:v -map 0:a"# -map 0:s"
−
        ffCommand=ffmpegBinary + " -i " + v.filename
+
        ffClip=" -ss 16 -t 100"
−
        ffCommand+=ffDefault + ffVideoCodec + ffAudioCodec + ffRate + ffSize + ffMaps + ffClip + ffOutput
+
        ffClip=""
−
    elif v.resolution == (720,576):
+
        ffCommand=ffmpegBinary + " -i " + v.filename
−
        debug(s, "Compression profile for720")
+
        ffCommand+=ffDefault + ffVideoCodec + ffAudioCodec + ffRate + ffSize + ffMaps + ffClip + ffOutput
−
        ffDefault=" -c copy"
+
    elif v.resolution == (720,576):
−
        ffVideoCodec=" -c:v libx264 -crf:v 23 -preset:v slow -tune:v film"
+
        debug(s, "Compression profile for720")
−
        ffAudioCodec=" -c:a libvorbis -q:a 5"
+
        ffDefault=" -c copy"
−
        #ffSize=" -s 960x540"
+
        ffVideoCodec=" -c:v libx264 -crf:v 23 -preset:v slow -tune:v film"
−
        ffSize=""
+
        ffAudioCodec=" -c:a libvorbis -q:a 5"
−
        ffMaps=" -map 0:v -map 0:a"# -map 0:s"
+
        #ffSize=" -s 960x540"
−
        ffClip=" -ss 16 -t 100"
+
        ffSize=""
−
        ffClip=""
+
        ffMaps=" -map 0:v -map 0:a"# -map 0:s"
−
        ffCommand=ffmpegBinary + " -i " + v.filename
+
        ffClip=" -ss 16 -t 100"
−
        ffCommand+=ffDefault + ffVideoCodec + ffAudioCodec + ffSize + ffMaps + ffClip + ffOutput
+
        ffClip=""
−
    else:
+
        ffCommand=ffmpegBinary + " -i " + v.filename
−
        error(s, "unsupported format")
+
        ffCommand+=ffDefault + ffVideoCodec + ffAudioCodec + ffSize + ffMaps + ffClip + ffOutput
−
+
    else:
−
+
        error(s, "unsupported format")
−
    #################################################################
+
 
−
    # compress video file
+
 
−
    cmd=shlex.split(ffCommand)
+
    #################################################################
−
    debug(s, "compressing video file...")
+
    # compress video file
−
    debug(s, "FFMPEG command:\n" + ffCommand)
+
    cmd=shlex.split(ffCommand)
−
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
    debug(s, "compressing video file...")
−
    stdout, stderr = p.communicate()
+
    debug(s, "FFMPEG command:\n" + ffCommand)
−
+
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
−
    print "Returncode:",p.returncode
+
    stdout, stderr = p.communicate()
−
    if os.path.isfile(v.tmpFilename):
+
 
−
        debug(s, "Video created.")
+
    print "Returncode:",p.returncode
−
    else:
+
    if os.path.isfile(v.tmpFilename):
−
        print stdout
+
        debug(s, "Video created.")
−
        print stderr
+
    else:
−
        error(s, "No video created.")
+
        print stdout
−
+
        print stderr
−
    #################################################################
+
        error(s, "No video created.")
−
    # check new filesize
+
 
−
    try:
+
    #################################################################
−
        v.newFilesize=os.path.getsize(v.tmpFilename)
+
    # check new filesize
−
    except OSError, e:
+
    try:
−
        error(s, "Could not get new filesize.")
+
        v.newFilesize=os.path.getsize(v.tmpFilename)
−
+
    except OSError, e:
−
    #################################################################
+
        error(s, "Could not get new filesize.")
−
    # compare video duration of result video
+
 
−
    v.newVideoInfo=getVideoStreamInformation(s, v.tmpFilename)
+
    #################################################################
−
    v.newDuration=getVideoDuration(v.newVideoInfo)
+
    # compare video duration of result video
−
    v.newNumAudioStreams=getNumberOfAudioStreams(v.newVideoInfo)
+
    v.newVideoInfo=getVideoStreamInformation(s, v.tmpFilename)
−
    difference=v.duration - v.newDuration
+
    v.newDuration=getVideoDuration(v.newVideoInfo)
−
    debug(s, "Duration difference                : " + str(difference))
+
    v.newNumAudioStreams=getNumberOfAudioStreams(v.newVideoInfo)
−
    if math.fabs(difference) > 2:
+
    difference=v.duration - v.newDuration
−
        debug(s, "Duration of original video: " + str(v.duration))
+
    debug(s, "Duration difference                : " + str(difference))
−
        debug(s, "Duration of new video    : " + str(v.newDuration))
+
    if math.fabs(difference) > 2:
−
        error(s, "Duration of result video is not equal.")
+
        debug(s, "Duration of original video: " + str(v.duration))
−
+
        debug(s, "Duration of new video    : " + str(v.newDuration))
−
    #################################################################
+
        error(s, "Duration of result video is not equal.")
−
    # all good ...
+
 
−
    # ... move compressed video file to destination
+
    #################################################################
−
    debug(s, "moving " + v.tmpFilename + " to " + v.newFilename)
+
    # all good ...
−
    try:  
+
    # ... move compressed video file to destination
−
        shutil.move(v.tmpFilename,v.newFilename)
+
    debug(s, "moving " + v.tmpFilename + " to " + v.newFilename)
−
    except shutil.Error, e:
+
    try:  
−
        print "Could not move compressed video file"
+
        shutil.move(v.tmpFilename,v.newFilename)
−
        error(s, e)
+
    except shutil.Error, e:
−
+
        print "Could not move compressed video file"
−
    if os.path.isfile(v.newFilename):
+
        error(s, e)
−
        debug(s, "Video successfully moved.")
+
 
−
    else:
+
    if os.path.isfile(v.newFilename):
−
        error(s, "Moved videofile does not exist.")
+
        debug(s, "Video successfully moved.")
−
    #################################################################
+
    else:
−
    # rename file in database
+
        error(s, "Moved videofile does not exist.")
−
    updateBase2="UPDATE recorded SET basename='%s', filesize='%s' WHERE chanid='%s' AND starttime='%s';"
+
    #################################################################
−
    updateBase2=updateBase2 % (v.newBasename, v.newFilesize, v.channelId, v.channelStart)
+
    # rename file in database
−
    # TODO
+
    updateBase2="UPDATE recorded SET basename='%s', filesize='%s' WHERE chanid='%s' AND starttime='%s';"
−
    # write transcoded="1"
+
    updateBase2=updateBase2 % (v.newBasename, v.newFilesize, v.channelId, v.channelStart)
−
    #  
+
    # TODO
−
    debug(s, updateBase2)
+
    # write transcoded="1"
−
    cursor.execute(updateBase2)
+
    #  
−
+
    debug(s, updateBase2)
−
    #################################################################
+
    cursor.execute(updateBase2)
−
    # delete pngs  
+
 
−
    #  
+
    #################################################################
−
    debug(s, "deleting... " + v.filename + "*png")
+
    # delete pngs  
−
    for filePath in glob.glob(v.filename + "*png"):
+
    #  
−
        if os.path.isfile(filePath):
+
    debug(s, "deleting... " + v.filename + "*png")
−
            debug(s, "delete " + filePath)
+
    for filePath in glob.glob(v.filename + "*png"):
−
            os.remove(filePath)
+
        if os.path.isfile(filePath):
−
+
            debug(s, "delete " + filePath)
−
    if os.path.isfile(v.filename):
+
            os.remove(filePath)
−
        debug(s, "deleting original videofile: " + v.filename)
+
 
−
        os.remove(v.filename)
+
    if os.path.isfile(v.filename):
−
    else:
+
        debug(s, "deleting original videofile: " + v.filename)
−
        error(s, "Could not delete original video")
+
        os.remove(v.filename)
−
+
    else:
−
    #################################################################
+
        error(s, "Could not delete original video")
−
    clean(s)
+
 
−
    #sys.exit(0)
+
    #################################################################
−
def allowedTimes():
+
    clean(s)
−
    now = datetime.now()
+
    #sys.exit(0)
−
    hour=int(datetime.strftime(now, "%H"))
+
def allowedTimes():
−
    if hour > 0 and not (hour > 17):
+
    now = datetime.now()
−
        return True
+
    hour=int(datetime.strftime(now, "%H"))
−
    else:
+
    if hour > 0 and not (hour > 17):
−
        return False
+
        return True
−
+
    else:
−
+
        return False
−
s=status()
+
 
−
s.prefix="system: "
+
 
−
# cleaning up temporary directory
+
s=status()
−
for filePath in glob.glob(TEMPPARENTDIR + "/*"):
+
s.prefix="system: "
−
    if os.path.isdir(filePath):
+
# cleaning up temporary directory
−
        debug(s, "delete old temporary directory " + filePath)
+
for filePath in glob.glob(TEMPPARENTDIR + "/*"):
−
        shutil.rmtree(filePath)
+
    if os.path.isdir(filePath):
−
+
        debug(s, "delete old temporary directory " + filePath)
−
count=0
+
        shutil.rmtree(filePath)
−
#while count < 1:
+
 
−
while True:
+
count=0
−
    s=status()
+
#while count < 1:
−
    s.prefix="system: "
+
while True:
−
    TEMPDIR=createTemp(s, TEMPPARENTDIR)
+
    s=status()
−
    s.tempDir=TEMPDIR
+
    s.prefix="system: "
−
+
    TEMPDIR=createTemp(s, TEMPPARENTDIR)
−
    if not enoughSpaceAvailable(TEMPDIR, space=freeSpaceInTempRequired):
+
    s.tempDir=TEMPDIR
−
        error(s, "Not enough space free in temporary directory!!")
+
 
−
    atexit.register(clean, status=s)
+
    if not enoughSpaceAvailable(TEMPDIR, space=freeSpaceInTempRequired):
−
+
        error(s, "Not enough space free in temporary directory!!")
−
    ############################
+
    atexit.register(clean, status=s)
−
    # video compression logic  
+
 
−
    if allowedTimes():
+
    ############################
−
        count+=1
+
    # video compression logic  
−
        debug(s, "Video number: " + str(count))
+
    if allowedTimes():
−
        compressVideo(s)
+
        count+=1
−
+
        debug(s, "Video number: " + str(count))
−
    time.sleep(600)
+
        compressVideo(s)
 +
 
 +
    time.sleep(600)
 +
</pre>
 +
}}

Revision as of 18:48, 21 October 2013

Due to disc restrictions in my mythbox i decided to write a script to compress all my videos. Mythtranscode does this but the videos have been recorded quite some time ago and are spread over multiple discs. Compressing those videos to a single destination is no option because they probably won't fit onto one partition. Therefore i wrote a script that dynamically picks a show that is at least two weeks old, compresses it and saves it to a partition with enough disc space. So, there are multiple folders where videos are waiting for compression and multiple folders where compressed videos are stored. Actually, these can be the same, but they don't have to be as long they are visible to mythtv (see mythtv-setup and storing directories). For compressed videos i created another storing group called 'compressed' and added all destination folders. Additionally i wanted this script to run as a system service and between 1 am and 6 pm.

Processing:

  1. Is it in the right time slot? (1am until 6pm)
  2. Pick a video for compression
  3. Temporarily rename video in mysql database
  4. Compress video in temporary folder
  5. Check if compressed video is okay (duration check)
  6. Update mysql database
  7. Back to 1.


This script consists of three program files: mythCompress.py, mythLogging.py, mythFunctions.py and one system script: compressVideos


PythonIcon.png compressVideos.py 

#!/usr/bin/env python

import MySQLdb
import os, sys, glob
import shlex, subprocess
import tempfile
import time
from datetime import datetime, timedelta
from os.path import basename
import re
import shutil
import math
import atexit

from mythFunctions import *
from mythLogging import error, debug, clean

import psutil
os.nice(19)
program = psutil.Process(os.getpid())
program.set_ionice(psutil.IOPRIO_CLASS_IDLE)

######################################################################
# SETTINGS
#
# folders to save compressed videos
DESTINATION=[ "/media/myth/compressed/" , "/media/data/mythtv/compressed/" ]
# folders to read uncompressed videos
SOURCES=[ "/media/data/mythtv/default/" , "/media/myth/default/" ] 
TEMPPARENTDIR=os.path.expanduser("~/mythtv-tmp/")
PRINTDEBUG=True
mysql_opts = {
    'host': "localhost",
    'user': "mythtv",
    'pass': "CHANGEME",
    'db':   "mythconverg"
    } 
minimumAge = timedelta(days=14) # for conversion
ffmpegBinary="/home/rfs/bin/ffmpeg"
ffprobeBinary="/home/rfs/bin/ffprobe"

freeSpaceRequired=20000000000
freeSpaceInTempRequired=10000000000
######################################################################
# constants
PID=os.getpgid(0)
now=datetime.now()

#################################################################
# functions
def createTemp(status, parentDirectory):
    tempDir=None
    if os.path.exists(parentDirectory):
        tempDir = tempfile.mkdtemp(prefix=parentDirectory + str(PID) + "-")
        debug(status, "Temporary directory: " + tempDir)
    else:
        status.tempDir=None
        error(status, "Parent directory for temp does not exist.")
    # change directory to working directory
    try:
        os.chdir(tempDir)
    except OSError:
        status.tempDir=None
        error(status, "Could not find temporary directory (OSError).")
    # return result
    return tempDir

class status(object):
    prefix=None
    tempDir=None

class videoFile(object):
    channelId=None
    channelStart=None
    channelDate=None
    title=None

    # quellvideo
    filename=None
    basename=None
    videoInfo=None
    duration=None
    numAudioStreams=None

    # temporaere ausgabe
    tmpFilename=None
    tmpBasename=None

    # ziel
    newFilename=None
    newBasename=None
    newVideoInfo=None
    newDuration=None
    newNumAudioStreams=None
    

def compressVideo(s):
    #################################################################
    # choose partition to save compressed video
    # a partition with enough free space is required
    chosenDestination=None
    for d in DESTINATION:
        if enoughSpaceAvailable(d, space=freeSpaceRequired):
            chosenDestination=d
            break
    if chosenDestination is None:
        error(s, "Could not select a partition with enough free space")
    else:
        debug(s, "Chose following partition as destination for compressed video:\n\t" 
              + chosenDestination)

    TEMPDIR=s.tempDir
    #################################################################
    # choose video file for compression
    v=videoFile()
    v.filename=selectVideoFile(SOURCES, minimumAge)
    if v.filename is None:
        error(s, "No videofile could be selected.")
    v.basename=basename(v.filename)
    debug(s, "Chose videofile: " + v.filename)

    try:
        v.filesize=os.path.getsize(v.filename)
    except OSError, e:
        error(s, "Could not get mpg filesize.")
    
    # prefix for LOGGING
    s.prefix=v.basename
    tokens=v.basename.replace(".mpg","").split("_")

    if len(tokens) != 2:
        error(s, "Could not parse filename for chanid and starttime.")

    try:
        v.channelId=int(tokens[0])
        v.channelStart=tokens[1]
        v.channelDate=datetime.strptime(v.channelStart, "%Y%m%d%H%M%S")
    except ValueError:
        error(s, "Could not parse channel id and date: " + "_".join(tokens))
    # get video information
    v.videoInfo=getVideoStreamInformation(s, v.filename)
    v.resolution=getVideoResolution(v.videoInfo)
    v.duration=getVideoDuration(v.videoInfo)
    v.numAudioStreams=getNumberOfAudioStreams(v.videoInfo)
    
    v.creation=getFileCreationDate(v.filename)

    v.newBasename=str(v.channelId) + "_" + str(v.channelStart) + ".avi"
    v.newFilename=chosenDestination + v.newBasename

    v.tmpBasename=str(v.channelId) + "_" + str(v.channelStart) + ".tmp"
    v.tmpFilename=TEMPDIR + "/" + v.newBasename


    debug(s, str(v.channelId)  + "," + str(v.channelStart) + "," + str(v.channelDate) + ":\n"
          + "filename:\t" + v.filename + "\n"
          + "resolution:\t" + 'x'.join(map(str,v.resolution)) + "\n"
          )
    debug(s, "Temporary Directory: " + TEMPDIR)
    debug(s, "Temporary filename : " + v.tmpFilename)
    debug(s, "Temporary basename : " + v.tmpBasename )
    debug(s, "New basename       : " + v.newBasename )
    debug(s, "New File will be   : " + v.newFilename)


    #################################################################
    # update mysql db
    mysql = MySQLdb.connect(mysql_opts['host'], mysql_opts['user'], mysql_opts['pass'], mysql_opts['db']) 
    cursor = mysql.cursor()
    sql="SELECT title FROM recorded WHERE chanid='%s' AND starttime='%s'"
    sql=sql % (v.channelId , v.channelDate.strftime('%Y-%m-%d %H:%M:%S'))
    debug(s, "Mysql database query for title")
    debug(s, sql)
    cursor.execute(sql)
    #cursor.execute(sql, (v.channelId,  v.channelDate.strftime('%Y-%m-%d %H:%M:%S')))
    videoTable=cursor.fetchall()
    try:
        v.title=videoTable[0][0]
        if len(v.title) == 0:
            error(s, "Title has zero length")
    except IndexError, e:
        error(s, "No mysql database entry found for this video.")

    debug(s, "Title of video is  : " + v.title)

    # fix seeking and bookmarking
    debug(s, "fix Seeking and bookmarking")
    delete1="DELETE FROM `recordedseek` WHERE chanid='%s' AND starttime='%s'"
    delete1=delete1 % (v.channelId, v.channelStart)
    debug(s, delete1)
    cursor.execute(delete1)#, (v.channelId, v.channelStart))

    delete2="DELETE FROM `recordedmarkup` WHERE chanid='%s' AND starttime='%s'"
    delete2=delete2 % (v.channelId, v.channelStart)
    debug(s, delete2)
    cursor.execute(delete2)#, (v.channelId, v.channelStart))

    updateBase1="UPDATE `recorded` SET basename='%s' WHERE chanid='%s' AND starttime='%s';"
    debug(s, "\nrenaming file in database")
    updateBase1=updateBase1 % (v.tmpBasename, v.channelId, v.channelStart)
    debug(s, updateBase1)
    cursor.execute(updateBase1)#, (v.tmpBasename, v.channelId, v.channelStart))

    #################################################################
    # set conversion parameter
    ffCommand=""
    ffOutput=" " + v.tmpFilename
    # TODO remove ffClip
    if v.resolution == (1280,720):
        debug(s, "Compression profile for 1280")
        ffDefault=" -c copy"
        ffVideoCodec=" -c:v libx264 -crf:v 23 -preset:v slow -tune:v film"
        ffAudioCodec=" -c:a libvorbis -q:a 5"
        ffSize=" -s 960x540"
        ffRate=" -r 25"
        ffMaps=" -map 0:v -map 0:a"# -map 0:s"
        ffClip=" -ss 16 -t 100"
        ffClip=""
        ffCommand=ffmpegBinary + " -i " + v.filename
        ffCommand+=ffDefault + ffVideoCodec + ffAudioCodec + ffRate + ffSize + ffMaps + ffClip + ffOutput
    elif v.resolution == (720,576):
        debug(s, "Compression profile for720")
        ffDefault=" -c copy"
        ffVideoCodec=" -c:v libx264 -crf:v 23 -preset:v slow -tune:v film"
        ffAudioCodec=" -c:a libvorbis -q:a 5"
        #ffSize=" -s 960x540"
        ffSize=""
        ffMaps=" -map 0:v -map 0:a"# -map 0:s"
        ffClip=" -ss 16 -t 100"
        ffClip=""
        ffCommand=ffmpegBinary + " -i " + v.filename
        ffCommand+=ffDefault + ffVideoCodec + ffAudioCodec + ffSize + ffMaps + ffClip + ffOutput
    else:
        error(s, "unsupported format")


    #################################################################
    # compress video file
    cmd=shlex.split(ffCommand)
    debug(s, "compressing video file...")
    debug(s, "FFMPEG command:\n" + ffCommand)
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    stdout, stderr = p.communicate()

    print "Returncode:",p.returncode
    if os.path.isfile(v.tmpFilename):
        debug(s, "Video created.")
    else:
        print stdout
        print stderr
        error(s, "No video created.")

    #################################################################
    # check new filesize
    try:
        v.newFilesize=os.path.getsize(v.tmpFilename)
    except OSError, e:
        error(s, "Could not get new filesize.")

    #################################################################
    # compare video duration of result video
    v.newVideoInfo=getVideoStreamInformation(s, v.tmpFilename)
    v.newDuration=getVideoDuration(v.newVideoInfo)
    v.newNumAudioStreams=getNumberOfAudioStreams(v.newVideoInfo)
    difference=v.duration - v.newDuration
    debug(s, "Duration difference                : " + str(difference))
    if math.fabs(difference) > 2:
        debug(s, "Duration of original video: " + str(v.duration))
        debug(s, "Duration of new video     : " + str(v.newDuration))
        error(s, "Duration of result video is not equal.")

    #################################################################
    # all good ...
    # ... move compressed video file to destination
    debug(s, "moving " + v.tmpFilename + " to " + v.newFilename)
    try: 
        shutil.move(v.tmpFilename,v.newFilename)
    except shutil.Error, e:
        print "Could not move compressed video file"
        error(s, e)

    if os.path.isfile(v.newFilename):
        debug(s, "Video successfully moved.")
    else:
        error(s, "Moved videofile does not exist.")
    #################################################################
    # rename file in database
    updateBase2="UPDATE recorded SET basename='%s', filesize='%s' WHERE chanid='%s' AND starttime='%s';"
    updateBase2=updateBase2 % (v.newBasename, v.newFilesize, v.channelId, v.channelStart)
    # TODO
    # write transcoded="1"
    # 
    debug(s, updateBase2)
    cursor.execute(updateBase2)

    #################################################################
    # delete pngs 
    # 
    debug(s, "deleting... " + v.filename + "*png")
    for filePath in glob.glob(v.filename + "*png"):
        if os.path.isfile(filePath):
            debug(s, "delete " + filePath)
            os.remove(filePath)

    if os.path.isfile(v.filename):
        debug(s, "deleting original videofile: " + v.filename)
        os.remove(v.filename)
    else:
        error(s, "Could not delete original video")

    #################################################################
    clean(s)
    #sys.exit(0)
def allowedTimes():
    now = datetime.now()
    hour=int(datetime.strftime(now, "%H"))
    if hour > 0 and not (hour > 17):
        return True
    else:
        return False


s=status()
s.prefix="system: "
# cleaning up temporary directory
for filePath in glob.glob(TEMPPARENTDIR + "/*"):
    if os.path.isdir(filePath):
        debug(s, "delete old temporary directory " + filePath)
        shutil.rmtree(filePath)

count=0
#while count < 1:
while True:
    s=status()
    s.prefix="system: "
    TEMPDIR=createTemp(s, TEMPPARENTDIR)
    s.tempDir=TEMPDIR

    if not enoughSpaceAvailable(TEMPDIR, space=freeSpaceInTempRequired):
        error(s, "Not enough space free in temporary directory!!")
    atexit.register(clean, status=s)

    ############################
    # video compression logic 
    if allowedTimes():
        count+=1
        debug(s, "Video number: " + str(count))
        compressVideo(s)

    time.sleep(600)
Retrieved from "http://www.mythtv.org/wiki?title=MythCompress&oldid=59331"