Difference between revisions of "MythCompress"

From MythTV Official Wiki
Jump to: navigation, search
m (additional wiki formatting)
 
Line 535: Line 535:
 
}}
 
}}
  
'''compressVideos init script'''
+
{{Code_box|compressVideos|
 
<pre>
 
<pre>
 
#! /bin/sh
 
#! /bin/sh
Line 569: Line 569:
  
 
</pre>
 
</pre>
 +
}}

Latest revision as of 19:23, 21 October 2013

Due to disc restrictions in my mythbox i decided to write a script to compress all my videos. Mythtranscode does this but the videos have been recorded quite some time ago and are spread over multiple discs. Compressing those videos to a single destination is no option because they probably won't fit onto one partition. Therefore i wrote a script that dynamically picks a show that is at least two weeks old, compresses it and saves it to a partition with enough disc space. So, there are multiple folders where videos are waiting for compression and multiple folders where compressed videos are stored. Actually, these folders can be the same, but they don't have to be as long they are visible to mythtv (see mythtv-setup and storing directories). For compressed videos i created another storing group called 'compressed' and added all destination folders. Videos are recorded in two different formats by my tv-card (PAL and 720p). This script decides upon video resolution which parameters to pick. Additionally i wanted this script to run as a system service and between 1 am and 6 pm.

Processing:

  1. Is it in the right time slot? (1am until 6pm)
  2. Pick a video for compression
  3. Temporarily rename video in mysql database
  4. Compress video in temporary folder
  5. Check if compressed video is okay (duration check)
  6. Update mysql database
  7. Back to 1.

Following programs are required: python 2.7, ffmpeg (ffmpeg version git-2013-10-01-2e2a2d8), ffprobe (comes with ffmpeg)

This script consists of three python scripts: mythCompress.py, mythLogging.py, mythFunctions.py and one system (init.d) script: compressVideos

mythCompress.py

PythonIcon.png mythCompress.py

#!/usr/bin/env python

import MySQLdb
import os, sys, glob
import shlex, subprocess
import tempfile
import time
from datetime import datetime, timedelta
from os.path import basename
import re
import shutil
import math
import atexit

from mythFunctions import *
from mythLogging import error, debug, clean

import psutil
os.nice(19)
program = psutil.Process(os.getpid())
program.set_ionice(psutil.IOPRIO_CLASS_IDLE)

######################################################################
# SETTINGS
#
# folders to save compressed videos
DESTINATION=[ "/media/myth/compressed/" , "/media/data/mythtv/compressed/" ]
# folders to read uncompressed videos
SOURCES=[ "/media/data/mythtv/default/" , "/media/myth/default/" ] 
TEMPPARENTDIR=os.path.expanduser("~/mythtv-tmp/")
PRINTDEBUG=True
mysql_opts = {
    'host': "localhost",
    'user': "mythtv",
    'pass': "CHANGEME",
    'db':   "mythconverg"
    } 
minimumAge = timedelta(days=14) # for conversion
ffmpegBinary="/home/rfs/bin/ffmpeg"
ffprobeBinary="/home/rfs/bin/ffprobe"

freeSpaceRequired=20000000000
freeSpaceInTempRequired=10000000000
######################################################################
# constants
PID=os.getpgid(0)
now=datetime.now()

#################################################################
# functions
def createTemp(status, parentDirectory):
    tempDir=None
    if os.path.exists(parentDirectory):
        tempDir = tempfile.mkdtemp(prefix=parentDirectory + str(PID) + "-")
        debug(status, "Temporary directory: " + tempDir)
    else:
        status.tempDir=None
        error(status, "Parent directory for temp does not exist.")
    # change directory to working directory
    try:
        os.chdir(tempDir)
    except OSError:
        status.tempDir=None
        error(status, "Could not find temporary directory (OSError).")
    # return result
    return tempDir

class status(object):
    prefix=None
    tempDir=None

class videoFile(object):
    channelId=None
    channelStart=None
    channelDate=None
    title=None

    # quellvideo
    filename=None
    basename=None
    videoInfo=None
    duration=None
    numAudioStreams=None

    # temporaere ausgabe
    tmpFilename=None
    tmpBasename=None

    # ziel
    newFilename=None
    newBasename=None
    newVideoInfo=None
    newDuration=None
    newNumAudioStreams=None
    

def compressVideo(s):
    #################################################################
    # choose partition to save compressed video
    # a partition with enough free space is required
    chosenDestination=None
    for d in DESTINATION:
        if enoughSpaceAvailable(d, space=freeSpaceRequired):
            chosenDestination=d
            break
    if chosenDestination is None:
        error(s, "Could not select a partition with enough free space")
    else:
        debug(s, "Chose following partition as destination for compressed video:\n\t" 
              + chosenDestination)

    TEMPDIR=s.tempDir
    #################################################################
    # choose video file for compression
    v=videoFile()
    v.filename=selectVideoFile(SOURCES, minimumAge)
    if v.filename is None:
        error(s, "No videofile could be selected.")
    v.basename=basename(v.filename)
    debug(s, "Chose videofile: " + v.filename)

    try:
        v.filesize=os.path.getsize(v.filename)
    except OSError, e:
        error(s, "Could not get mpg filesize.")
    
    # prefix for LOGGING
    s.prefix=v.basename
    tokens=v.basename.replace(".mpg","").split("_")

    if len(tokens) != 2:
        error(s, "Could not parse filename for chanid and starttime.")

    try:
        v.channelId=int(tokens[0])
        v.channelStart=tokens[1]
        v.channelDate=datetime.strptime(v.channelStart, "%Y%m%d%H%M%S")
    except ValueError:
        error(s, "Could not parse channel id and date: " + "_".join(tokens))
    # get video information
    v.videoInfo=getVideoStreamInformation(s, v.filename)
    v.resolution=getVideoResolution(v.videoInfo)
    v.duration=getVideoDuration(v.videoInfo)
    v.numAudioStreams=getNumberOfAudioStreams(v.videoInfo)
    
    v.creation=getFileCreationDate(v.filename)

    v.newBasename=str(v.channelId) + "_" + str(v.channelStart) + ".avi"
    v.newFilename=chosenDestination + v.newBasename

    v.tmpBasename=str(v.channelId) + "_" + str(v.channelStart) + ".tmp"
    v.tmpFilename=TEMPDIR + "/" + v.newBasename


    debug(s, str(v.channelId)  + "," + str(v.channelStart) + "," + str(v.channelDate) + ":\n"
          + "filename:\t" + v.filename + "\n"
          + "resolution:\t" + 'x'.join(map(str,v.resolution)) + "\n"
          )
    debug(s, "Temporary Directory: " + TEMPDIR)
    debug(s, "Temporary filename : " + v.tmpFilename)
    debug(s, "Temporary basename : " + v.tmpBasename )
    debug(s, "New basename       : " + v.newBasename )
    debug(s, "New File will be   : " + v.newFilename)


    #################################################################
    # update mysql db
    mysql = MySQLdb.connect(mysql_opts['host'], mysql_opts['user'], mysql_opts['pass'], mysql_opts['db']) 
    cursor = mysql.cursor()
    sql="SELECT title FROM recorded WHERE chanid='%s' AND starttime='%s'"
    sql=sql % (v.channelId , v.channelDate.strftime('%Y-%m-%d %H:%M:%S'))
    debug(s, "Mysql database query for title")
    debug(s, sql)
    cursor.execute(sql)
    #cursor.execute(sql, (v.channelId,  v.channelDate.strftime('%Y-%m-%d %H:%M:%S')))
    videoTable=cursor.fetchall()
    try:
        v.title=videoTable[0][0]
        if len(v.title) == 0:
            error(s, "Title has zero length")
    except IndexError, e:
        error(s, "No mysql database entry found for this video.")

    debug(s, "Title of video is  : " + v.title)

    # fix seeking and bookmarking
    debug(s, "fix Seeking and bookmarking")
    delete1="DELETE FROM `recordedseek` WHERE chanid='%s' AND starttime='%s'"
    delete1=delete1 % (v.channelId, v.channelStart)
    debug(s, delete1)
    cursor.execute(delete1)#, (v.channelId, v.channelStart))

    delete2="DELETE FROM `recordedmarkup` WHERE chanid='%s' AND starttime='%s'"
    delete2=delete2 % (v.channelId, v.channelStart)
    debug(s, delete2)
    cursor.execute(delete2)#, (v.channelId, v.channelStart))

    updateBase1="UPDATE `recorded` SET basename='%s' WHERE chanid='%s' AND starttime='%s';"
    debug(s, "\nrenaming file in database")
    updateBase1=updateBase1 % (v.tmpBasename, v.channelId, v.channelStart)
    debug(s, updateBase1)
    cursor.execute(updateBase1)#, (v.tmpBasename, v.channelId, v.channelStart))

    #################################################################
    # set conversion parameter
    ffCommand=""
    ffOutput=" " + v.tmpFilename
    # TODO remove ffClip
    if v.resolution == (1280,720):
        debug(s, "Compression profile for 1280")
        ffDefault=" -c copy"
        ffVideoCodec=" -c:v libx264 -crf:v 23 -preset:v slow -tune:v film"
        ffAudioCodec=" -c:a libvorbis -q:a 5"
        ffSize=" -s 960x540"
        ffRate=" -r 25"
        ffMaps=" -map 0:v -map 0:a"# -map 0:s"
        ffClip=" -ss 16 -t 100"
        ffClip=""
        ffCommand=ffmpegBinary + " -i " + v.filename
        ffCommand+=ffDefault + ffVideoCodec + ffAudioCodec + ffRate + ffSize + ffMaps + ffClip + ffOutput
    elif v.resolution == (720,576):
        debug(s, "Compression profile for720")
        ffDefault=" -c copy"
        ffVideoCodec=" -c:v libx264 -crf:v 23 -preset:v slow -tune:v film"
        ffAudioCodec=" -c:a libvorbis -q:a 5"
        #ffSize=" -s 960x540"
        ffSize=""
        ffMaps=" -map 0:v -map 0:a"# -map 0:s"
        ffClip=" -ss 16 -t 100"
        ffClip=""
        ffCommand=ffmpegBinary + " -i " + v.filename
        ffCommand+=ffDefault + ffVideoCodec + ffAudioCodec + ffSize + ffMaps + ffClip + ffOutput
    else:
        error(s, "unsupported format")


    #################################################################
    # compress video file
    cmd=shlex.split(ffCommand)
    debug(s, "compressing video file...")
    debug(s, "FFMPEG command:\n" + ffCommand)
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    stdout, stderr = p.communicate()

    print "Returncode:",p.returncode
    if os.path.isfile(v.tmpFilename):
        debug(s, "Video created.")
    else:
        print stdout
        print stderr
        error(s, "No video created.")

    #################################################################
    # check new filesize
    try:
        v.newFilesize=os.path.getsize(v.tmpFilename)
    except OSError, e:
        error(s, "Could not get new filesize.")

    #################################################################
    # compare video duration of result video
    v.newVideoInfo=getVideoStreamInformation(s, v.tmpFilename)
    v.newDuration=getVideoDuration(v.newVideoInfo)
    v.newNumAudioStreams=getNumberOfAudioStreams(v.newVideoInfo)
    difference=v.duration - v.newDuration
    debug(s, "Duration difference                : " + str(difference))
    if math.fabs(difference) > 2:
        debug(s, "Duration of original video: " + str(v.duration))
        debug(s, "Duration of new video     : " + str(v.newDuration))
        error(s, "Duration of result video is not equal.")

    #################################################################
    # all good ...
    # ... move compressed video file to destination
    debug(s, "moving " + v.tmpFilename + " to " + v.newFilename)
    try: 
        shutil.move(v.tmpFilename,v.newFilename)
    except shutil.Error, e:
        print "Could not move compressed video file"
        error(s, e)

    if os.path.isfile(v.newFilename):
        debug(s, "Video successfully moved.")
    else:
        error(s, "Moved videofile does not exist.")
    #################################################################
    # rename file in database
    updateBase2="UPDATE recorded SET basename='%s', filesize='%s' WHERE chanid='%s' AND starttime='%s';"
    updateBase2=updateBase2 % (v.newBasename, v.newFilesize, v.channelId, v.channelStart)
    # TODO
    # write transcoded="1"
    # 
    debug(s, updateBase2)
    cursor.execute(updateBase2)

    #################################################################
    # delete pngs 
    # 
    debug(s, "deleting... " + v.filename + "*png")
    for filePath in glob.glob(v.filename + "*png"):
        if os.path.isfile(filePath):
            debug(s, "delete " + filePath)
            os.remove(filePath)

    if os.path.isfile(v.filename):
        debug(s, "deleting original videofile: " + v.filename)
        os.remove(v.filename)
    else:
        error(s, "Could not delete original video")

    #################################################################
    clean(s)
    #sys.exit(0)
def allowedTimes():
    now = datetime.now()
    hour=int(datetime.strftime(now, "%H"))
    if hour > 0 and not (hour > 17):
        return True
    else:
        return False


s=status()
s.prefix="system: "
# cleaning up temporary directory
for filePath in glob.glob(TEMPPARENTDIR + "/*"):
    if os.path.isdir(filePath):
        debug(s, "delete old temporary directory " + filePath)
        shutil.rmtree(filePath)

count=0
#while count < 1:
while True:
    s=status()
    s.prefix="system: "
    TEMPDIR=createTemp(s, TEMPPARENTDIR)
    s.tempDir=TEMPDIR

    if not enoughSpaceAvailable(TEMPDIR, space=freeSpaceInTempRequired):
        error(s, "Not enough space free in temporary directory!!")
    atexit.register(clean, status=s)

    ############################
    # video compression logic 
    if allowedTimes():
        count+=1
        debug(s, "Video number: " + str(count))
        compressVideo(s)

    time.sleep(600)


mythFunctions.py

PythonIcon.png mythFunctions.py

#!/usr/bin/env python

import json
import shlex, subprocess
import MySQLdb
import os, sys, glob
import tempfile
import time
from datetime import datetime, timedelta
from os.path import basename
import re
import shutil


PID=os.getpgid(0)
ffprobeBinary="/home/rfs/bin/ffprobe"

from mythLogging import error

#######################################################################################
def getDisksize(filename):
    df = subprocess.Popen(["/bin/df", filename], stdout=subprocess.PIPE)
    output = df.communicate()[0]
    print output
    device, size, used, available, percent, mountpoint = output.split("\n")[1].split()
    
def get_fs_freespace(pathname):
    "Get the free space of the filesystem containing pathname"
    stat= os.statvfs(pathname)
    # use f_bfree for superuser, or f_bavail if filesystem
    # has reserved space for superuser
    return int(stat.f_bfree*stat.f_bsize)

def enoughSpaceAvailable(path, space=20000000000):
    return get_fs_freespace(path) > space

def getFileCreationDate(filename):
    t = os.path.getmtime(filename)
    then=datetime.fromtimestamp(t)
    return then

def isMPGvideoPath(path):
    return os.path.isfile(path) and path.endswith(".mpg")

def isMPGvideoFile(f, directory):
    filename=directory + "/" + f
    return os.path.isfile(filename) and f.endswith(".mpg")

# video suchen, das aelter als minimumAge ist
def selectVideoFile(SOURCES, minimumAge):
    result=None
    now = datetime.now()
    for directory in SOURCES:
        files = [f for f in os.listdir( directory ) if isMPGvideoFile(f, directory)]
        for f in files:
            filename=directory + "/" + f
            then=getFileCreationDate(filename)
            if (now - then) > minimumAge:
                return filename
    return result

def getVideoStreamInformation(status, pathtovideo):
    stdout = ""
    stderr = ""
    try:
        cmd=ffprobeBinary + " -print_format json -show_streams -loglevel quiet "
        cmd+=pathtovideo
        p = subprocess.Popen(shlex.split(cmd),
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)
        stdout, stderr = p.communicate()
    except OSError:
        error(status, "Could not get video duration with ffprobe.")
    if len(stdout) == 0:
        error(status, "No data read for file: " + pathtovideo)

    data = json.loads(stdout)
    streams=None
    try:
        streams=data['streams']
    except KeyError, e:
        error(status, "Could not load stream information.")
    return streams

def getVideoDuration(videoInfo):
    duration=None
    for s in videoInfo:
        try:
            codec_type=s['codec_type']
            if codec_type=="video":
                d=s['duration']
                if len(d) > 0:
                    duration=float(d)
                    break
        except KeyError, e:
            pass
    return duration

def getVideoResolution(videoInfo):
    w=0; h=0
    for s in videoInfo:
        try:
            codec_type=s['codec_type']
            if codec_type=="video":
                w=int(s['width'])
                h=int(s['height'])
                break
        except KeyError, e:
            pass
    return (w,h)

def getNumberOfAudioStreams(videoInfo):
    count=0
    for s in videoInfo:
        try:
            codec_type=s['codec_type']
            if codec_type=="audio":
                count=count+1
        except KeyError, e:
            pass
    return count

mythLogging.py

PythonIcon.png mythLogging.py

#!/usr/bin/env python
import sys,os, shutil

PRINTDEBUG=True

def debug(status, message):
    if PRINTDEBUG:
        for line in message.split("\n"):
            print "STATUS:" + status.prefix + " " + line

def error(status, message):
    for line in message.split("\n"):
        sys.stderr.write("ERROR: " + status.prefix + " " + str(line) + "\n")
        print "ERROR: " + status.prefix + " " + str(line)
    clean(status)
    sys.exit(1)

def clean(status):
    os.chdir(os.path.expanduser("~"))
    tempDir=status.tempDir
    if not tempDir is None:
        # remove temporary directory
        if os.path.exists(tempDir):
            debug(status, "Removing temporary directory " + tempDir + ".")
            shutil.rmtree(tempDir)


Script.png compressVideos

#! /bin/sh
#
# /etc/init.d/vncserver         this Script
# /usr/bin/vncserver            Program
#

# Check for missing binaries
AUTOSSH_BIN=/home/rfs/bin/mythCompress.py
test -x $AUTOSSH_BIN || exit 5

LOG="/home/rfs/logs/mythCompress.syslog"

case "$1" in
    start)
        echo "Starting automatic video compression"
        sudo -u rfs /home/rfs/bin/mythCompress.py >> $LOG 2>> $LOG &
        echo "Started"
        ;;
#
     stop)
        echo "Stopping automatic video compression not implemented"
        echo 'ps axu | grep "mythCompress" | grep -v "init.d" | grep -v grep | awk '{print $2}' | xargs kill'
        ;;
#
#
    *)
        echo "Usage: $0 {start|stop}"
        exit 1
        ;;
esac