MythCompress
From MythTV Official Wiki
Due to disc restrictions in my mythbox i decided to write a script to compress all my videos. Mythtranscode does this but the videos have been recorded quite some time ago and are spread over multiple discs. Compressing those videos to a single destination is no option because they probably won't fit onto one partition. Therefore i wrote a script that dynamically picks a show that is at least two weeks old, compresses it and saves it to a partition with enough disc space. So, there are multiple folders where videos are waiting for compression and multiple folders where compressed videos are stored. Actually, these can be the same, but they don't have to be as long they are visible to mythtv (see mythtv-setup and storing directories). For compressed videos i created another storing group called 'compressed' and added all destination folders. Additionally i wanted this script to run as a system service and between 1 am and 6 pm.
Processing:
1.) Is it in the right time slot? (1am until 6pm)
2.) Pick a video for compression
3.) Temporarily rename video in mysql database
3.) Compress video in temporary folder
4.) Check if compressed video is okay (duration check)
5.) Update mysql database
6.) Back to 1.)
This script consists of three program files: mythCompress.py, mythLogging.py, mythFunctions.py
and one system script: compressVideos
#!/usr/bin/env python
import MySQLdb
import os, sys, glob
import shlex, subprocess
import tempfile
import time
from datetime import datetime, timedelta
from os.path import basename
import re
import shutil
import math
import atexit
from mythFunctions import *
from mythLogging import error, debug, clean
import psutil
os.nice(19)
program = psutil.Process(os.getpid())
program.set_ionice(psutil.IOPRIO_CLASS_IDLE)
######################################################################
# SETTINGS
#
# folders to save compressed videos
DESTINATION=[ "/media/myth/compressed/" , "/media/data/mythtv/compressed/" ]
# folders to read uncompressed videos
SOURCES=[ "/media/data/mythtv/default/" , "/media/myth/default/" ]
TEMPPARENTDIR=os.path.expanduser("~/mythtv-tmp/")
PRINTDEBUG=True
mysql_opts = {
'host': "localhost",
'user': "mythtv",
'pass': "CHANGEME",
'db': "mythconverg"
}
minimumAge = timedelta(days=14) # for conversion
ffmpegBinary="/home/rfs/bin/ffmpeg"
ffprobeBinary="/home/rfs/bin/ffprobe"
freeSpaceRequired=20000000000
freeSpaceInTempRequired=10000000000
######################################################################
# constants
PID=os.getpgid(0)
now=datetime.now()
#################################################################
# functions
def createTemp(status, parentDirectory):
tempDir=None
if os.path.exists(parentDirectory):
tempDir = tempfile.mkdtemp(prefix=parentDirectory + str(PID) + "-")
debug(status, "Temporary directory: " + tempDir)
else:
status.tempDir=None
error(status, "Parent directory for temp does not exist.")
# change directory to working directory
try:
os.chdir(tempDir)
except OSError:
status.tempDir=None
error(status, "Could not find temporary directory (OSError).")
# return result
return tempDir
class status(object):
prefix=None
tempDir=None
class videoFile(object):
channelId=None
channelStart=None
channelDate=None
title=None
# quellvideo
filename=None
basename=None
videoInfo=None
duration=None
numAudioStreams=None
# temporaere ausgabe
tmpFilename=None
tmpBasename=None
# ziel
newFilename=None
newBasename=None
newVideoInfo=None
newDuration=None
newNumAudioStreams=None
def compressVideo(s):
#################################################################
# choose partition to save compressed video
# a partition with enough free space is required
chosenDestination=None
for d in DESTINATION:
if enoughSpaceAvailable(d, space=freeSpaceRequired):
chosenDestination=d
break
if chosenDestination is None:
error(s, "Could not select a partition with enough free space")
else:
debug(s, "Chose following partition as destination for compressed video:\n\t"
+ chosenDestination)
TEMPDIR=s.tempDir
#################################################################
# choose video file for compression
v=videoFile()
v.filename=selectVideoFile(SOURCES, minimumAge)
if v.filename is None:
error(s, "No videofile could be selected.")
v.basename=basename(v.filename)
debug(s, "Chose videofile: " + v.filename)
try:
v.filesize=os.path.getsize(v.filename)
except OSError, e:
error(s, "Could not get mpg filesize.")
# prefix for LOGGING
s.prefix=v.basename
tokens=v.basename.replace(".mpg","").split("_")
if len(tokens) != 2:
error(s, "Could not parse filename for chanid and starttime.")
try:
v.channelId=int(tokens[0])
v.channelStart=tokens[1]
v.channelDate=datetime.strptime(v.channelStart, "%Y%m%d%H%M%S")
except ValueError:
error(s, "Could not parse channel id and date: " + "_".join(tokens))
# get video information
v.videoInfo=getVideoStreamInformation(s, v.filename)
v.resolution=getVideoResolution(v.videoInfo)
v.duration=getVideoDuration(v.videoInfo)
v.numAudioStreams=getNumberOfAudioStreams(v.videoInfo)
v.creation=getFileCreationDate(v.filename)
v.newBasename=str(v.channelId) + "_" + str(v.channelStart) + ".avi"
v.newFilename=chosenDestination + v.newBasename
v.tmpBasename=str(v.channelId) + "_" + str(v.channelStart) + ".tmp"
v.tmpFilename=TEMPDIR + "/" + v.newBasename
debug(s, str(v.channelId) + "," + str(v.channelStart) + "," + str(v.channelDate) + ":\n"
+ "filename:\t" + v.filename + "\n"
+ "resolution:\t" + 'x'.join(map(str,v.resolution)) + "\n"
)
debug(s, "Temporary Directory: " + TEMPDIR)
debug(s, "Temporary filename : " + v.tmpFilename)
debug(s, "Temporary basename : " + v.tmpBasename )
debug(s, "New basename : " + v.newBasename )
debug(s, "New File will be : " + v.newFilename)
#################################################################
# update mysql db
mysql = MySQLdb.connect(mysql_opts['host'], mysql_opts['user'], mysql_opts['pass'], mysql_opts['db'])
cursor = mysql.cursor()
sql="SELECT title FROM recorded WHERE chanid='%s' AND starttime='%s'"
sql=sql % (v.channelId , v.channelDate.strftime('%Y-%m-%d %H:%M:%S'))
debug(s, "Mysql database query for title")
debug(s, sql)
cursor.execute(sql)
#cursor.execute(sql, (v.channelId, v.channelDate.strftime('%Y-%m-%d %H:%M:%S')))
videoTable=cursor.fetchall()
try:
v.title=videoTable[0][0]
if len(v.title) == 0:
error(s, "Title has zero length")
except IndexError, e:
error(s, "No mysql database entry found for this video.")
debug(s, "Title of video is : " + v.title)
# fix seeking and bookmarking
debug(s, "fix Seeking and bookmarking")
delete1="DELETE FROM `recordedseek` WHERE chanid='%s' AND starttime='%s'"
delete1=delete1 % (v.channelId, v.channelStart)
debug(s, delete1)
cursor.execute(delete1)#, (v.channelId, v.channelStart))
delete2="DELETE FROM `recordedmarkup` WHERE chanid='%s' AND starttime='%s'"
delete2=delete2 % (v.channelId, v.channelStart)
debug(s, delete2)
cursor.execute(delete2)#, (v.channelId, v.channelStart))
updateBase1="UPDATE `recorded` SET basename='%s' WHERE chanid='%s' AND starttime='%s';"
debug(s, "\nrenaming file in database")
updateBase1=updateBase1 % (v.tmpBasename, v.channelId, v.channelStart)
debug(s, updateBase1)
cursor.execute(updateBase1)#, (v.tmpBasename, v.channelId, v.channelStart))
#################################################################
# set conversion parameter
ffCommand=""
ffOutput=" " + v.tmpFilename
# TODO remove ffClip
if v.resolution == (1280,720):
debug(s, "Compression profile for 1280")
ffDefault=" -c copy"
ffVideoCodec=" -c:v libx264 -crf:v 23 -preset:v slow -tune:v film"
ffAudioCodec=" -c:a libvorbis -q:a 5"
ffSize=" -s 960x540"
ffRate=" -r 25"
ffMaps=" -map 0:v -map 0:a"# -map 0:s"
ffClip=" -ss 16 -t 100"
ffClip=""
ffCommand=ffmpegBinary + " -i " + v.filename
ffCommand+=ffDefault + ffVideoCodec + ffAudioCodec + ffRate + ffSize + ffMaps + ffClip + ffOutput
elif v.resolution == (720,576):
debug(s, "Compression profile for720")
ffDefault=" -c copy"
ffVideoCodec=" -c:v libx264 -crf:v 23 -preset:v slow -tune:v film"
ffAudioCodec=" -c:a libvorbis -q:a 5"
#ffSize=" -s 960x540"
ffSize=""
ffMaps=" -map 0:v -map 0:a"# -map 0:s"
ffClip=" -ss 16 -t 100"
ffClip=""
ffCommand=ffmpegBinary + " -i " + v.filename
ffCommand+=ffDefault + ffVideoCodec + ffAudioCodec + ffSize + ffMaps + ffClip + ffOutput
else:
error(s, "unsupported format")
#################################################################
# compress video file
cmd=shlex.split(ffCommand)
debug(s, "compressing video file...")
debug(s, "FFMPEG command:\n" + ffCommand)
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
print "Returncode:",p.returncode
if os.path.isfile(v.tmpFilename):
debug(s, "Video created.")
else:
print stdout
print stderr
error(s, "No video created.")
#################################################################
# check new filesize
try:
v.newFilesize=os.path.getsize(v.tmpFilename)
except OSError, e:
error(s, "Could not get new filesize.")
#################################################################
# compare video duration of result video
v.newVideoInfo=getVideoStreamInformation(s, v.tmpFilename)
v.newDuration=getVideoDuration(v.newVideoInfo)
v.newNumAudioStreams=getNumberOfAudioStreams(v.newVideoInfo)
difference=v.duration - v.newDuration
debug(s, "Duration difference : " + str(difference))
if math.fabs(difference) > 2:
debug(s, "Duration of original video: " + str(v.duration))
debug(s, "Duration of new video : " + str(v.newDuration))
error(s, "Duration of result video is not equal.")
#################################################################
# all good ...
# ... move compressed video file to destination
debug(s, "moving " + v.tmpFilename + " to " + v.newFilename)
try:
shutil.move(v.tmpFilename,v.newFilename)
except shutil.Error, e:
print "Could not move compressed video file"
error(s, e)
if os.path.isfile(v.newFilename):
debug(s, "Video successfully moved.")
else:
error(s, "Moved videofile does not exist.")
#################################################################
# rename file in database
updateBase2="UPDATE recorded SET basename='%s', filesize='%s' WHERE chanid='%s' AND starttime='%s';"
updateBase2=updateBase2 % (v.newBasename, v.newFilesize, v.channelId, v.channelStart)
# TODO
# write transcoded="1"
#
debug(s, updateBase2)
cursor.execute(updateBase2)
#################################################################
# delete pngs
#
debug(s, "deleting... " + v.filename + "*png")
for filePath in glob.glob(v.filename + "*png"):
if os.path.isfile(filePath):
debug(s, "delete " + filePath)
os.remove(filePath)
if os.path.isfile(v.filename):
debug(s, "deleting original videofile: " + v.filename)
os.remove(v.filename)
else:
error(s, "Could not delete original video")
#################################################################
clean(s)
#sys.exit(0)
def allowedTimes():
now = datetime.now()
hour=int(datetime.strftime(now, "%H"))
if hour > 0 and not (hour > 17):
return True
else:
return False
s=status()
s.prefix="system: "
# cleaning up temporary directory
for filePath in glob.glob(TEMPPARENTDIR + "/*"):
if os.path.isdir(filePath):
debug(s, "delete old temporary directory " + filePath)
shutil.rmtree(filePath)
count=0
#while count < 1:
while True:
s=status()
s.prefix="system: "
TEMPDIR=createTemp(s, TEMPPARENTDIR)
s.tempDir=TEMPDIR
if not enoughSpaceAvailable(TEMPDIR, space=freeSpaceInTempRequired):
error(s, "Not enough space free in temporary directory!!")
atexit.register(clean, status=s)
############################
# video compression logic
if allowedTimes():
count+=1
debug(s, "Video number: " + str(count))
compressVideo(s)
time.sleep(600)
