Difference between revisions of "Mythcommflag-wrapper"

From MythTV Official Wiki
Jump to: navigation, search
(Improvements: - added some things for people to hack on)
m (Add version 0.28 icon)
 
(24 intermediate revisions by 9 users not shown)
Line 4: Line 4:
 
|long=A simple bash script based on [[Silence-detect.sh]] that wraps mythcommflag and can be used manually (via 'Begin Advert Detection' menu option in recording browser) to use silence detection for commflagging for certain UK TV channels.
 
|long=A simple bash script based on [[Silence-detect.sh]] that wraps mythcommflag and can be used manually (via 'Begin Advert Detection' menu option in recording browser) to use silence detection for commflagging for certain UK TV channels.
 
|category=Scripts
 
|category=Scripts
|S24=yes}}
+
|S24=yes|S26=yes|S27=yes|S28=yes}}
  
=== Rename the existing mythcommflag binary ===
+
=== Prerequisites ===
  
Mythbackend runs the commflag differently according to the configured name (i.e <code>JobQueueCommFlagCommand</code> in the <code>settings</code> table). If it's not '<code>mythcommflag</code>', then it always runs the configured commflagger with no parameters (i.e. in "flag all uncommflagged recordings"). Therefore it's necessary for this script to replace and wrap <code>mythcommflag</code>. e.g.
+
This script needs <code>mytharchivehelper</code> which may be included in the MythArchive package ( No longer needed for 0.26 ).Ensure that it is installed on your machine. It also needs [[http://www.ffmpeg.org/<code>ffmpeg</code>]] and [[http://mp3splt.sourceforge.net/<code>mp3splt</code>]]. It's likely that these are packaged for your distro, either directly, or by a third-party repository (e.g. ATrpms for Fedora).
 
 
<code><pre>
 
mv /usr/bin/mythcommflag /usr/bin/mythcommflag.orig
 
</pre></code>
 
  
 
=== Install wrapper script in place of mythcommflag ===
 
=== Install wrapper script in place of mythcommflag ===
  
Install this script as /usr/bin/mythcommflag (or wherever your original mythcommflag binary was located)
+
Install this script to somewhere accessible by the backend.  In mythtv-setup, the command used to run for commercial detection can be modified.  The '''''mythcommflag''''' option is a special phrase, telling the jobqueue to run
 +
  ${PREFIX}/bin/mythcommflag -j %JOBID% -V %VERBOSELEVEL%
 +
If you define any string other than '''''mythcommflag''''', the command options must be explicitly defined, such as
 +
  /home/mythtv/bin/mythcommflag-wrapper.sh -j %JOBID% -V 0
  
 
{{Code box|mythcommflag-wrapper.sh|
 
{{Code box|mythcommflag-wrapper.sh|
Line 49: Line 48:
 
echo >>$LOGFILE "No mysql.txt found in $HOME or $HOME/.mythtv - exiting!"
 
echo >>$LOGFILE "No mysql.txt found in $HOME or $HOME/.mythtv - exiting!"
 
exit 1
 
exit 1
 +
fi
 +
 +
MAH=`which mytharchivehelper`
 +
if [ "$MAH" = "" ]; then
 +
        echo "mytharchivehelper not found in $PATH - exiting!"
 +
        exit 1
 
fi
 
fi
  
Line 81: Line 86:
  
 
# AJB20110129 Have already checked for pre-existing cutlist, and this check doesn't work properly after --clearcutlist anyway
 
# AJB20110129 Have already checked for pre-existing cutlist, and this check doesn't work properly after --clearcutlist anyway
# [[ "Cutlist: "  == `mythcommflag.orig --getcutlist -f $filename  |grep Cutlist` ]] \
+
# [[ "Cutlist: "  == `mythcommflag --getcutlist -f $filename  |grep Cutlist` ]] \
 
#     || { echo already has cutlist && exit 1; }
 
#     || { echo already has cutlist && exit 1; }
  
Line 110: Line 115:
 
if [ $# -eq 0 ]; then
 
if [ $# -eq 0 ]; then
 
# run with no parameters, flag every unflagged recording
 
# run with no parameters, flag every unflagged recording
exec mythcommflag.orig
+
exec mythcommflag
 
exit $?
 
exit $?
 
else
 
else
Line 120: Line 125:
 
else
 
else
 
# we're being used in some other way, run the real mythcommmflag
 
# we're being used in some other way, run the real mythcommmflag
echo >>$LOGFILE "running mythcommflag.orig $*"
+
echo >>$LOGFILE "running mythcommflag $*"
exec mythcommflag.orig $*
+
exec mythcommflag $*
 
exit $?
 
exit $?
 
fi
 
fi
Line 178: Line 183:
 
else
 
else
 
# Not a whitelisted channel for silence_detect
 
# Not a whitelisted channel for silence_detect
echo >>$LOGFILE "won't run silence-detect, running mythcommflag.orig $*"
+
echo >>$LOGFILE "won't run silence-detect, running mythcommflag $*"
exec mythcommflag.orig $*
+
exec mythcommflag $*
 
exit $?
 
exit $?
 
fi
 
fi
Line 187: Line 192:
 
</pre>
 
</pre>
 
}}
 
}}
 +
 +
 +
 +
 +
Version using mythutil for 0.25+ and it no longer needs mythexport
 +
 +
{{Code box|mythcommflag-wrapper.sh|
 +
<pre>
 +
#!/bin/bash
 +
 +
# Version using mythutil for 0.25+ as it no longer needs mythexport: Tuned detect slightly, updated whitelist to include Freesat/Freeview +1 channels. Supports UK Freeview/Freesat HD Channels as they have MPEG audio channels. ( Channels with AC3 Only are not supported. ) Changed ffmpeg to avconv, as ffmpeg is deprecated. and ffmpeg -i occasionally chose the Audio Descriptive track, which is nearly all silence.
 +
 +
# edit/tune these #
 +
# Allow ad breaks to be upto 400s long by coalescing non-silence
 +
MAXCOMMBREAKSECS=400
 +
# -80dB and minimum of 0.12s to be considered 'silent'
 +
#MP3SPLT_OPTS="th=-70,min=0.15"
 +
MP3SPLT_OPTS="th=-80,min=0.12"
 +
# Log file
 +
LOGFILE="/var/log/mythtv/mythcommflag-wrapper"
 +
# Copy commercial skiplist to cutlist automatically? 1=Enabled
 +
#COPYTOCUTLIST=1
 +
 +
MYTHCFG=""
 +
# mythbackend in Fedora packages has $HOME=/etc/mythtv
 +
if [ -e $HOME/mysql.txt ]; then
 +
MYTHCFG="$HOME/mysql.txt"
 +
fi
 +
 +
 +
if [ -e $HOME/.mythtv/mysql.txt ]; then
 +
MYTHCFG="$HOME/.mythtv/mysql.txt"
 +
fi
 +
if [ "$MYTHCFG" = "" ]; then
 +
echo >>$LOGFILE "No mysql.txt found in $HOME or $HOME/.mythtv - exiting!"
 +
exit 1
 +
fi
 +
 +
# DB username and password
 +
MYTHHOST=`grep DBHostName <$MYTHCFG | awk -F= '{print $2}'`
 +
MYTHUSER=`grep DBUserName <$MYTHCFG | awk -F= '{print $2}'`
 +
MYTHPASS=`grep DBPassword <$MYTHCFG | awk -F= '{print $2}'`
 +
MYTHDB=`grep DBName <$MYTHCFG | awk -F= '{print $2}'`
 +
#echo >>$LOGFILE "[$MYTHHOST] [$MYTHUSER] [$MYTHPASS] [$MYTHDB]"
 +
 +
# root of MythTV recordings
 +
RECORDINGSROOT=`mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "select dirname from storagegroup;" $MYTHDB | tail -n +2 | sort | uniq | tr '\n' ' '`
 +
 +
 +
echo >>$LOGFILE "$0 run with [$*] at `date` by `whoami`"
 +
echo >>$LOGFILE "RECORDINGSROOT=$RECORDINGSROOT"
 +
 +
# From <http://www.mythtv.org/wiki/Silence-detect.sh>
 +
 +
silence_detect() {
 +
local filename=$1
 +
 +
TMPDIR=`mktemp -d /tmp/mythcommflag.XXXXXX` || exit 1
 +
 +
cd $TMPDIR
 +
touch `basename $filename`.touch
 +
ionice -c3 nice avconv -i $filename -acodec copy sound.mp2
 +
ionice -c3 nice mp3splt -s -p $MP3SPLT_OPTS sound.mp2
 +
 +
CUTLIST=`tail --lines=+3 mp3splt.log|sort -g |\
 +
      awk 'BEGIN{start=0;ORS=","}{if($2-start<'$MAXCOMMBREAKSECS')
 +
      {finish=$2} else {print int(start*25+1)"-"int(finish*25-25);
 +
      start=$1; finish=$2;}}END{print int(start*25+1)"-"9999999}'`
 +
echo >>$LOGFILE "silence-detect has generated cutlist: $CUTLIST"
 +
 +
echo >>$LOGFILE "silence-detect(): CHANID=$CHANID, STARTTIME=$STARTTIME"
 +
 +
rm -rf $TMPDIR
 +
}
 +
 +
 +
if [ $# -eq 0 ]; then
 +
# run with no parameters, flag every unflagged recording
 +
exec mythcommflag
 +
exit $?
 +
else
 +
if [ $# -eq 4 -a "$1" = "-j" -a "$3" = "-V" ]; then
 +
# this is a manual flag job
 +
# TODO AJB20110129 generate a filelist from recorded table where commflagged=0 and cutlist=0
 +
# then run silence_detect over each filename
 +
JOB=$2
 +
else
 +
# we're being used in some other way, run the real mythcommmflag
 +
echo >>$LOGFILE "running mythcommflag $*"
 +
exec mythcommflag $*
 +
exit $?
 +
fi
 +
echo >>$LOGFILE "running job $JOB"
 +
HASCUTLIST=`mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "select recorded.cutlist from recorded join jobqueue where jobqueue.id=$JOB and jobqueue.chanid=recorded.chanid and jobqueue.starttime=recorded.starttime;" $MYTHDB | tail -n +2`
 +
if [ "$HASCUTLIST" = "1" ]; then
 +
echo >>$LOGFILE "program already has (manual?) cutlist, exiting"
 +
exit 0
 +
fi
 +
CALLSIGN=`mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "select channel.callsign from channel join jobqueue where jobqueue.id=$JOB and jobqueue.chanid=channel.chanid;" $MYTHDB | tail -n +2`
 +
CHANID=`mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "select chanid from jobqueue where jobqueue.id=$JOB;" $MYTHDB | tail -n +2`
 +
STARTTIME=`mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "select starttime from jobqueue where jobqueue.id=$JOB;" $MYTHDB | tail -n +2`
 +
echo >>$LOGFILE "channel callsign is $CALLSIGN"
 +
echo >>$LOGFILE "chanid=$CHANID STARTTIME=$STARTTIME"
 +
BASENAME=`mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "select recorded.basename from recorded join jobqueue where jobqueue.id=$JOB and jobqueue.chanid=recorded.chanid and jobqueue.starttime=recorded.starttime;" $MYTHDB | tail -n +2`
 +
echo >>$LOGFILE "basename is $BASENAME"
 +
FILENAME=`ionice -c3 nice find ${RECORDINGSROOT} -name $BASENAME`
 +
echo >>$LOGFILE "filename is $FILENAME"
 +
 +
        # See discussion page for proposed changes to this next section
 +
 +
# make lower case and remove white space to account for minor
 +
# presentation changes between transmissions and over time callsigns
 +
# on Freesat and freeview differ e.g. "FIVE *" and "Five*". Also
 +
# removed is the +1 suffix from the criteria, if the primary channel
 +
# is allowable the +1 variant should also be included in white list.
 +
lowerCALLSIGN="$(echo $CALLSIGN | tr '[A-Z]' '[a-z]') | tr -d '+1' | tr -d '[:space:]'"
 +
 +
case $lowerCALLSIGN in # the following list should be in lower case and without any white space and no need to specify "+1" variants
 +
# E4/channel 4 treats trailers as program. The silence markers are
 +
# only either side of actual adverts. E4 often has trailers before and
 +
# after adverts, which will not be cut, and appear within the wanted show.
 +
# Works for other FIVE channels with caveat that they include news
 +
# bulletins which are't cut
 +
"five"|"5"|"fiveusa"|"5usa"|"channel4"|"channel4hd"|"more4"|"e4"|"film4"|"4seven"|"itv1"|"itv1hd"|"itv2"|"itv3"|"itv4"|"dave"|"davejavu")
 +
echo >>$LOGFILE "Callsign in whitelist - will run silence_detect"
 +
mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "update recorded set commflagged=2 where chanid=$CHANID and starttime='${STARTTIME}';" $MYTHDB
 +
CUTLIST=""
 +
echo >>$LOGFILE "silence_detect $FILENAME"
 +
silence_detect $FILENAME
 +
echo >>$LOGFILE "silence_detect() set CUTLIST to $CUTLIST"
 +
let BREAKS=`grep -o "-" <<< "$CUTLIST" | wc -l`
 +
echo >>$LOGFILE "$BREAKS break(s) found."
 +
                mythutil --setskiplist $CUTLIST --chanid="$CHANID" --starttime="${STARTTIME}"
 +
RC=$?
 +
echo >>$LOGFILE "mythutil setskiplist returned $RC"
 +
if [ $RC -eq 0 ]; then
 +
mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "update recorded set commflagged=1 where chanid=$CHANID and starttime='${STARTTIME}';" ${MYTHDB}
 +
mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "update jobqueue set status=272, comment='Finished, $BREAKS break(s) found.' where id=$JOB;" ${MYTHDB}
 +
if [ $COPYTOCUTLIST -eq 1 ]; then
 +
mythutil --gencutlist --chanid="$CHANID" --starttime="${STARTTIME}"
 +
RC=$?
 +
if [ $RC -eq 0 ]; then
 +
echo >>$LOGFILE "mythutil --gencutlist successfully copied skip list to cut list"
 +
else
 +
echo >>$LOGFILE "mythutil --gencutlist failed, returned $RC"
 +
fi
 +
fi
 +
else
 +
echo >>$LOGFILE "mythcommflag failed; returned $RC"
 +
mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "update recorded set commflagged=0 where chanid=$CHANID and starttime='${STARTTIME}';" ${MYTHDB}
 +
fi
 +
;;
 +
*)
 +
# Not a whitelisted channel for silence_detect
 +
echo >>$LOGFILE "won't run silence-detect, running mythcommflag $*"
 +
exec mythcommflag $*
 +
exit $?
 +
;;
 +
esac
 +
fi
 +
echo >>$LOGFILE
 +
</pre>
 +
}}
 +
 +
Info from trac Ticket {{ticket|11936}}:-
 +
<pre style="white-space: pre-wrap;
 +
white-space: -moz-pre-wrap;
 +
white-space: -pre-wrap;
 +
white-space: -o-pre-wrap;
 +
word-wrap: break-word;">
 +
 +
In the later version of mythcommflag-wrapper.sh (ver 0.25+) there is an  error in the lowerCALLSIGN. The ')' is too early and should be before the  final '"'
 +
</pre>
 +
 +
I'm no coder. I couldn't possibly comment.
 +
[[User:Stevegoodey|Stevegoodey]] 11:40, 12 November 2013 (UTC)
  
 
=== Edit script variables ===
 
=== Edit script variables ===
  
 
Edit the environment variables in the wrapper script according to your local installation.
 
Edit the environment variables in the wrapper script according to your local installation.
 
Note that you can specify multiple storage directories by separating them with a space. e.g.:
 
<code>RECORDINGSROOT="/media/disk/mythtv/recordings /mnt/nas/mythtv/recordings"</code>
 
  
 
=== Enable commflagging ===
 
=== Enable commflagging ===
  
Check that commflagging is allowed in mythtv-setup (i.e. the <code>JobAllowCommFlag</code> setting should be 1) and that the commflag command (i.e. <code>JobQueueCommFlagCommand</code>) is <code>mythcommflag</code>.
+
Check that commflagging is allowed in mythtv-setup (i.e. the <code>JobAllowCommFlag</code> setting should be 1).
  
 
=== Test ===
 
=== Test ===
Line 203: Line 381:
 
In the 'Watch Recordings' browser, select a recording that was from a channel included in the script's whitelist (Film4, Dave, Dave ja vu, ITV1-4, ITV1 +1, ITV2 +1), hit menu, then Jobs, then 'Begin Advert Detection'). Within a few minutes, that recording should be commflagged appropriately.
 
In the 'Watch Recordings' browser, select a recording that was from a channel included in the script's whitelist (Film4, Dave, Dave ja vu, ITV1-4, ITV1 +1, ITV2 +1), hit menu, then Jobs, then 'Begin Advert Detection'). Within a few minutes, that recording should be commflagged appropriately.
  
=== Improvements ===
+
=== Future Planned Improvements ===
  
 
<strike>Fill in the TODOs.</strike> Done, apart from automatic "flag all unflagged recordings" mode. Maybe when it can be trusted to not go haywire...
 
<strike>Fill in the TODOs.</strike> Done, apart from automatic "flag all unflagged recordings" mode. Maybe when it can be trusted to not go haywire...
 +
:: Unnecessary. Running ''mythcommflag --queue'' will queue all unflagged recordings for commercial flagging in the jobqueue.
  
 
<strike>Expand the channel whitelist ("Five" tested once and failed, others???)</strike>
 
<strike>Expand the channel whitelist ("Five" tested once and failed, others???)</strike>
Line 218: Line 397:
  
 
Merge this into the real mythcommflag. Will probably need a new column in the <code>channels</code> table to signify which commflagging method should be used, and that in turn will require some work to the channel-editor UI of <code>mythtv-setup</code>.
 
Merge this into the real mythcommflag. Will probably need a new column in the <code>channels</code> table to signify which commflagging method should be used, and that in turn will require some work to the channel-editor UI of <code>mythtv-setup</code>.
 +
 +
===For 0.27+. Change to blacklisting and using HD ===
 +
 +
By ceesquared. Many thanks to cowbutt for starting this.
 +
 +
This version changes the the script to  use blacklisting instead of whitelisting on the assumption that anything transmitted on Freesat/freeview will conform to the silence break requirement. It also incorporates the changes in 0.27 that removes mysql.txt and replaces it with config.xml. If you find a channel that does not conform then you can add it to the blacklist after examining the log output. It has been checked on 0.28.
 +
 +
{{Code box|mythcommflag-wrapper.sh|
 +
<pre>
 +
#!/bin/bash
 +
 +
# place in /usr/bin/
 +
# Version using mythutil for 0.25+ as it no longer needs mythexport: Only checked on Freesat HD but it should work with
 +
# Freeview HD if they have AC3 audio.  Changed ffmpeg to avconv, as ffmpeg is deprecated. avconv used the Audio Descriptive
 +
# track without the ac3 adjustment. On my test this was audio+ AD but I am told that it could be nearly all silence so I
 +
# forced the use of the ac3 track.
 +
# Edited version which assumes all freesat/freeview channels conform to the silent gap requirement. There is therefore no
 +
# need for a whitelist.
 +
# Updated for 0.27 as mysql.txt no longer exists.
 +
 +
# Function to extract data from config.xml
 +
 +
Extract(){
 +
Sourcefile=$1
 +
Searchterm=$2
 +
# $1 is filename, $2 is search item
 +
Filterterm="<"$Searchterm">"
 +
Result=`grep $Filterterm <$Sourcefile `
 +
Result=`echo ${Result//$Filterterm/}`
 +
Filterterm="</"$Searchterm">"
 +
Result=`echo ${Result//$Filterterm/}`
 +
echo $Result
 +
}
 +
#End Extract
 +
 +
# From <http://www.mythtv.org/wiki/Silence-detect.sh>
 +
 +
silence_detect() {
 +
local filename=$1
 +
 +
TMPDIR=`mktemp -d /tmp/mythcommflag.XXXXXX` || exit 1
 +
 +
cd $TMPDIR
 +
touch `basename $filename`.touch
 +
# Find out if ac3(HD)
 +
avprobe -show_streams -select_streams a $filename | grep ac3 >/dev/null
 +
if [ $? -eq 0 ];then # ac3 stream
 +
# use avconv to get stream location from stderr/stdout. Info is in stderr so it has to be piped to stdout to see it(2>&1).
 +
# find the line with ac3 This has the format    Stream #m:n[0xhex]...............
 +
streamline=$(avconv -hide_banner -i $filename 2>&1 | grep ac3)
 +
# Now strip that line to leave m:n
 +
# strip off leading characters
 +
stream1=${streamline#*#} # leaving m:n[0xhex]........
 +
# we now strip the trailing characters
 +
stream2=${stream1%[0x*} # gives m:n in stream2
 +
# export ac3 audio in mp2 format
 +
ionice -c 3 nice avconv -i $filename -map $stream2 -codec:a mp2 sound.mp2
 +
else ionice -c 3 nice avconv -i $filename -acodec copy sound.mp2
 +
fi
 +
ionice -c 3 nice mp3splt -s -p $MP3SPLT_OPTS sound.mp2
 +
 +
CUTLIST=`tail --lines=+3 mp3splt.log|sort -g |\
 +
      awk 'BEGIN{start=0;ORS=","}{if($2-start<'$MAXCOMMBREAKSECS')
 +
      {finish=$2} else {print int(start*25+1)"-"int(finish*25-25);
 +
      start=$1; finish=$2;}}END{print int(start*25+1)"-"9999999}'`
 +
echo "silence-detect has generated cutlist: $CUTLIST" >>$LOGFILE
 +
 +
echo "silence-detect(): CHANID=$CHANID, STARTTIME=$STARTTIME" >>$LOGFILE
 +
# remove the temporary directory and its contents
 +
rm -rf $TMPDIR
 +
}
 +
# End Silence detect
 +
 +
# Main body
 +
 +
# edit/tune these #
 +
# Allow ad breaks to be up to 400s long by coalescing non-silence
 +
MAXCOMMBREAKSECS=400
 +
# -80dB and minimum of 01.0s to be considered 'silent'
 +
# Limited investigation has shown that the advert gap is about 2 secs and the between adverts
 +
# gap about 1 sec
 +
#MP3SPLT_OPTS="th=-70,min=0.15"
 +
MP3SPLT_OPTS="th=-80,min=1.0"
 +
#MP3SPLT_OPTS="th=-80,min=0.50"
 +
#MP3SPLT_OPTS="th=-60,min=0.10"
 +
# Log file
 +
LOGFILE="/var/log/mythtv/mythcommflag.log"
 +
# Save passed parameters
 +
NOPARAM=$#
 +
PARAMSTR=$*
 +
if [ $NOPARAM -eq 4 -a "$1" = "-j" -a "$3" = "-v" ]; then
 +
# this is a manual flag job
 +
JOB=$2
 +
MANFLAG=true
 +
fi
 +
 +
echo "starting Run ukcommflag $date" >>$LOGFILE
 +
 +
# Copy commercial skiplist to cutlist automatically? 1=Enabled
 +
#COPYTOCUTLIST=1
 +
 +
MYTHCFG=""
 +
# Find config.xml in Ver 0.27 as mysql.txt no longer exists
 +
if [ -e $HOME/.mythtv/config.xml ]; then
 +
MYTHCFG="$HOME/.mythtv/config.xml"
 +
fi
 +
 +
 +
if [ -e $MYTHCONFDIR/config.xml ]; then
 +
MYTHCFG="$MYTHCONFDIR/config.xml"
 +
fi
 +
if [ "$MYTHCFG" = "" ]; then
 +
echo "No config.xml found in $MYTHCONFDIR or $HOME/.mythtv - exiting!" >>$LOGFILE
 +
exit 1
 +
fi
 +
 +
# DB username and password
 +
MYTHHOST=$(Extract $MYTHCFG "Host")
 +
MYTHUSER=$(Extract $MYTHCFG "UserName")
 +
MYTHPASS=$(Extract $MYTHCFG "Password")
 +
MYTHDB=$(Extract $MYTHCFG "DatabaseName")
 +
# echo "Host, Username, Password, and Database name extracted" >>$LOGFILE
 +
# echo $MYTHHOST"  "$MYTHUSER"  "$MYTHPASS"  "$MYTHDB >>$LOGFILE
 +
 +
# root of MythTV recordings
 +
RECORDINGSROOT=`mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "select dirname from storagegroup;" $MYTHDB | tail -n +2 | sort | uniq | tr '\n' ' '`
 +
 +
 +
echo "$0 run with [$PARAMSTR] at `date` by `whoami`" >>$LOGFILE
 +
echo "RECORDINGSROOT=$RECORDINGSROOT" >>$LOGFILE
 +
 +
 +
if [ NOPARAM -eq 0 ]; then
 +
# run with no parameters, flag every unflagged recording
 +
exec mythcommflag
 +
exit $?
 +
else
 +
if [ MANFLAG -ne true ]; then
 +
# we're being used in some other way, run the real mythcommmflag
 +
exec mythcommflag $PARAMSTR
 +
exit $?
 +
fi
 +
echo "running job $JOB" >>$LOGFILE
 +
HASCUTLIST=`mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "select recorded.cutlist from recorded join jobqueue where jobqueue.id=$JOB and jobqueue.chanid=recorded.chanid and jobqueue.starttime=recorded.starttime;" $MYTHDB | tail -n +2`
 +
if [ "$HASCUTLIST" = "1" ]; then
 +
echo "program already has (manual?) cutlist, exiting" >>$LOGFILE
 +
exit 0
 +
fi
 +
CALLSIGN=`mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "select channel.callsign from channel join jobqueue where jobqueue.id=$JOB and jobqueue.chanid=channel.chanid;" $MYTHDB | tail -n +2`
 +
CHANID=`mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "select chanid from jobqueue where jobqueue.id=$JOB;" $MYTHDB | tail -n +2`
 +
STARTTIME=`mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "select starttime from jobqueue where jobqueue.id=$JOB;" $MYTHDB | tail -n +2`
 +
echo "channel callsign is $CALLSIGN" >>$LOGFILE
 +
echo "chanid=$CHANID STARTTIME=$STARTTIME" >>$LOGFILE
 +
BASENAME=`mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "select recorded.basename from recorded join jobqueue where jobqueue.id=$JOB and jobqueue.chanid=recorded.chanid and jobqueue.starttime=recorded.starttime;" $MYTHDB | tail -n +2`
 +
echo "basename is $BASENAME" >>$LOGFILE
 +
FILENAME=`ionice -c 3 nice find ${RECORDINGSROOT} -name $BASENAME`
 +
echo "filename is $FILENAME" >>$LOGFILE
 +
 +
 +
# make lower case and remove white space to account for minor
 +
# presentation changes between transmissions and over time callsigns
 +
# on Freesat and freeview differ e.g. "FIVE *" and "Five*". Also
 +
# removed is the +1 suffix from the criteria, if the primary channel
 +
# is allowable the +1 variant should also be included in white list.
 +
        # Remove HD on the same logic as +1
 +
        # Convert to lower case
 +
lowerCALLSIGN="$(echo $CALLSIGN | tr '[A-Z]' '[a-z]')"
 +
        # Remove +1
 +
lowerCALLSIGN=`echo ${lowerCALLSIGN//+1/}`
 +
# Remove +24
 +
lowerCALLSIGN=`echo ${lowerCALLSIGN//+24/}`
 +
#Remove hd
 +
lowerCALLSIGN=`echo ${lowerCALLSIGN//hd/}`
 +
#Remove spaces
 +
lowerCALLSIGN="$(echo $lowerCALLSIGN | tr -d '[:space:]')"
 +
    case $lowerCALLSIGN in
 +
# the following list should be in lower case and without any white space and no need to specify "+1" or HD variants
 +
"xxx"|"yyy")
 +
# A Blacklisted channel for silence_detect
 +
echo "won't run silence-detect on $CALLSIGN, running mythcommflag $PARAMSTR" >>$LOGFILE
 +
exec mythcommflag $PARAMSTR
 +
exit $?
 +
;;
 +
*)
 +
# E4/channel 4 treats trailers as program. The silence markers are
 +
# only either side of actual adverts. E4 often has trailers before and
 +
# after adverts, which will not be cut, and appear within the wanted show.
 +
# Works for other FIVE channels with caveat that they include news
 +
# bulletins which are't cut
 +
 +
        echo "Callsign $CALLSIGN not in Blacklist - will run silence_detect" >>$LOGFILE
 +
        mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "update recorded set commflagged=2 where chanid=$CHANID and starttime='${STARTTIME}';" $MYTHDB
 +
        CUTLIST=""
 +
        silence_detect $FILENAME
 +
        echo "silence_detect() set CUTLIST to $CUTLIST" >>$LOGFILE
 +
        let BREAKS=`grep -o "-" <<< "$CUTLIST" | wc -l`
 +
        echo "$BREAKS break(s) found." >>$LOGFILE
 +
        mythutil --setskiplist $CUTLIST --chanid="$CHANID" --starttime="${STARTTIME}"
 +
        RC=$?
 +
        echo "mythutil setskiplist returned $RC" >>$LOGFILE
 +
        if [ $RC -eq 0 ]; then
 +
    mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "update recorded set commflagged=1 where chanid=$CHANID and starttime='${STARTTIME}';" ${MYTHDB}
 +
        mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "update jobqueue set status=272, comment='Finished, $BREAKS break(s) found.' where id=$JOB;" ${MYTHDB}
 +
    if [ $COPYTOCUTLIST -eq 1 ]; then
 +
mythutil --gencutlist --chanid="$CHANID" --starttime="${STARTTIME}"
 +
RC=$?
 +
if [ $RC -eq 0 ]; then
 +
echo "mythutil --gencutlist successfully copied skip list to cut list" >>$LOGFILE
 +
else
 +
echo "mythutil --gencutlist failed, returned $RC" >>$LOGFILE
 +
fi
 +
    fi
 +
        else
 +
echo "mythcommflag failed; returned $RC" >>$LOGFILE
 +
mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "update recorded set commflagged=0 where chanid=$CHANID and starttime='${STARTTIME}';" ${MYTHDB}
 +
        fi
 +
        ;;
 +
    esac
 +
fi
 +
echo "Run Complete" >>$LOGFILE
 +
</pre>
 +
}}
 +
 +
If you enable logging then make sure that you have write permission for the logfile.
 +
  
 
[[Category:User Job Scripts]]
 
[[Category:User Job Scripts]]

Latest revision as of 14:51, 22 September 2016


Author Cowbutt
Description A simple bash script based on Silence-detect.sh that wraps mythcommflag and can be used manually (via 'Begin Advert Detection' menu option in recording browser) to use silence detection for commflagging for certain UK TV channels.
Supports Version24.png  Version26.png Version27.png Version28.png 


Prerequisites

This script needs mytharchivehelper which may be included in the MythArchive package ( No longer needed for 0.26 ).Ensure that it is installed on your machine. It also needs [ffmpeg] and [mp3splt]. It's likely that these are packaged for your distro, either directly, or by a third-party repository (e.g. ATrpms for Fedora).

Install wrapper script in place of mythcommflag

Install this script to somewhere accessible by the backend. In mythtv-setup, the command used to run for commercial detection can be modified. The mythcommflag option is a special phrase, telling the jobqueue to run

 ${PREFIX}/bin/mythcommflag -j %JOBID% -V %VERBOSELEVEL%

If you define any string other than mythcommflag, the command options must be explicitly defined, such as

 /home/mythtv/bin/mythcommflag-wrapper.sh -j %JOBID% -V 0


Script.png mythcommflag-wrapper.sh

#!/bin/bash


# edit/tune these #
# Allow ad breaks to be upto 400s long by coalescing non-silence
MAXCOMMBREAKSECS=400
# mytharchivehelper isn't frame-accurate, and mythcommflag won't add final cut pair if the end of the cut is greater
# than the actual final frame, so subtract $ENDFUDGEFRAMES from the value mytharchivehelper returns
ENDFUDGEFRAMES=75
# -70dB and minimum of 0.15s to be considered 'silent'
MP3SPLT_OPTS="th=-70,min=0.15"
# Log file
LOGFILE="/var/log/mythtv/mythcommflag-wrapper"
# Copy commercial skiplist to cutlist automatically? 1=Enabled
COPYTOCUTLIST=1

MYTHCFG=""
# mythbackend in Fedora packages has $HOME=/etc/mythtv
if [ -e $HOME/mysql.txt ]; then
	MYTHCFG="$HOME/mysql.txt"
fi


if [ -e $HOME/.mythtv/mysql.txt ]; then
	MYTHCFG="$HOME/.mythtv/mysql.txt"
fi
if [ "$MYTHCFG" = "" ]; then
	echo >>$LOGFILE "No mysql.txt found in $HOME or $HOME/.mythtv - exiting!"
	exit 1
fi

MAH=`which mytharchivehelper` 
if [ "$MAH" = "" ]; then
        echo "mytharchivehelper not found in $PATH - exiting!"
        exit 1
fi

# DB username and password
MYTHHOST=`grep DBHostName <$MYTHCFG | awk -F= '{print $2}'`
MYTHUSER=`grep DBUserName <$MYTHCFG | awk -F= '{print $2}'`
MYTHPASS=`grep DBPassword <$MYTHCFG | awk -F= '{print $2}'`
MYTHDB=`grep DBName <$MYTHCFG | awk -F= '{print $2}'`
#echo >>$LOGFILE "[$MYTHHOST] [$MYTHUSER] [$MYTHPASS] [$MYTHDB]"

# root of MythTV recordings
RECORDINGSROOT=`mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "select dirname from storagegroup;" $MYTHDB | tail -n +2 | sort | uniq | tr '\n' ' '`


echo >>$LOGFILE "$0 run with [$*] at `date` by `whoami`"
echo >>$LOGFILE "RECORDINGSROOT=$RECORDINGSROOT"
#export >>$LOGFILE

#/usr/bin/mythcommflag run with [-j 584 -V 8213] at Sat Jan 29 17:26:58 GMT 2011 by root
#select recorded.basename from recorded join jobqueue where jobqueue.id=584 and jobqueue.chanid=recorded.chanid and jobqueue.starttime=recorded.starttime;

silence_detect() {
		local filename=$1
		
		TMPDIR=`mktemp -d /tmp/mythcommflag.XXXXXX` || exit 1		
		
		# get frame count, less 3 seconds (75 frames) fudge factor
		let FRAMES=`mytharchivehelper -i $filename $TMPDIR/streaminfo.xml 1 2>&1 | grep " frames = " | awk -F"= " '{print $2}'`-${ENDFUDGEFRAMES}
		echo >>$LOGFILE "total frames = $FRAMES (+/- 75)"		

		# From <http://www.mythtv.org/wiki/Silence-detect.sh>

# AJB20110129 Have already checked for pre-existing cutlist, and this check doesn't work properly after --clearcutlist anyway
#		[[ "Cutlist: "  == `mythcommflag --getcutlist -f $filename  |grep Cutlist` ]] \
#		    || { echo already has cutlist && exit 1; }

		cd $TMPDIR
		touch `basename $filename`.touch
		ionice -c3 nice ffmpeg -i $filename -acodec copy sound.mp3
		ionice -c3 nice mp3splt -s -p $MP3SPLT_OPTS sound.mp3

		#AJB20110129 add '$FRAMES' because mythcommflag now requires *pairs* of frames
		CUTLIST=`tail --lines=+3 mp3splt.log|sort -g |\
		       awk 'BEGIN{start=0;ORS=","}{if($2-start<'$MAXCOMMBREAKSECS')
		       {finish=$2} else {print int(start*25+1)"-"int(finish*25-25);
		       start=$1; finish=$2;}}END{print int(start*25+1)"-"'$FRAMES'}'`
		echo >>$LOGFILE "silence-detect has generated cutlist: $CUTLIST"

		echo >>$LOGFILE "silence-detect(): CHANID=$CHANID, STARTTIME=$STARTTIME, FRAMES=$FRAMES"		
		CUTLIST=`tail --lines=+3 mp3splt.log|sort -g |\
		awk 'BEGIN{start=0;ORS="\n"}{if($2-start<'$MAXCOMMBREAKSECS') {finish=$2} else {print "INSERT INTO recordedmarkup (chanid,starttime,mark,type) VALUES ('$CHANID',\"'"$STARTTIME"'\","int(start*25+1)",4);\nINSERT INTO recordedmarkup (chanid,starttime,mark,type) VALUES ('$CHANID',\"'"$STARTTIME"'\","int(finish*25-25)",5);"; start=$1; finish=$2;}}END{print "INSERT INTO recordedmarkup (chanid,starttime,mark,type) VALUES ('$CHANID',\"'"$STARTTIME"'\","int(start*25+1)",4);\nINSERT INTO recordedmarkup (chanid,starttime,mark,type) VALUES ('$CHANID',\"'"$STARTTIME"'\",'$FRAMES',5);"}' | sed 's/\"/'\''/g'`
		RC=$?
#		echo >>$LOGFILE "silence-detect SQLgen returned with $RC, generated SQL skipist: $CUTLIST"		

#		CUTLIST="USE $MYTHDB; $CUTLIST"

		rm -rf $TMPDIR
}		


if [ $# -eq 0 ]; then
	# run with no parameters, flag every unflagged recording
	exec mythcommflag
	exit $?
else
	if [ $# -eq 4 -a "$1" = "-j" -a "$3" = "-V" ]; then
		# this is a manual flag job
		# TODO AJB20110129 generate a filelist from recorded table where commflagged=0 and cutlist=0
		# then run silence_detect over each filename
		JOB=$2
	else
		# we're being used in some other way, run the real mythcommmflag
		echo >>$LOGFILE "running mythcommflag $*"
		exec mythcommflag $*
		exit $?
	fi
	echo >>$LOGFILE "running job $JOB"
	HASCUTLIST=`mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "select recorded.cutlist from recorded join jobqueue where jobqueue.id=$JOB and jobqueue.chanid=recorded.chanid and jobqueue.starttime=recorded.starttime;" $MYTHDB | tail -n +2`	
	if [ "$HASCUTLIST" = "1" ]; then
		echo >>$LOGFILE "program already has (manual?) cutlist, exiting"
		exit 0
	fi
	CALLSIGN=`mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "select channel.callsign from channel join jobqueue where jobqueue.id=$JOB and jobqueue.chanid=channel.chanid;" $MYTHDB | tail -n +2`
	CHANID=`mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "select chanid from jobqueue where jobqueue.id=$JOB;" $MYTHDB | tail -n +2`	
	STARTTIME=`mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "select starttime from jobqueue where jobqueue.id=$JOB;" $MYTHDB | tail -n +2`
	echo >>$LOGFILE "channel callsign is $CALLSIGN"
	echo >>$LOGFILE "chanid=$CHANID STARTTIME=$STARTTIME"
	BASENAME=`mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "select recorded.basename from recorded join jobqueue where jobqueue.id=$JOB and jobqueue.chanid=recorded.chanid and jobqueue.starttime=recorded.starttime;" $MYTHDB | tail -n +2`	
	echo >>$LOGFILE "basename is $BASENAME"
	FILENAME=`ionice -c3 nice find ${RECORDINGSROOT} -name $BASENAME`
	echo >>$LOGFILE "filename is $FILENAME"	

#safe list
#	if [ "$CALLSIGN" = "Film4" -o "$CALLSIGN" = "ITV1" -o "$CALLSIGN" = "ITV1 +1" -o "$CALLSIGN" = "ITV2" -o "$CALLSIGN" = "ITV2 +1" -o "$CALLSIGN" = "ITV3" -o "$CALLSIGN" = "ITV4" -o "$CALLSIGN" = "Dave" -o "$CALLSIGN" = "Dave ja vu" ]; then
# Channel 4 channels seem to work too, but sometimes last cut is too long?
#	if [ "$CALLSIGN" = "Channel 4" -o "$CALLSIGN" = "Channel 4+1" -o "$CALLSIGN" = "More 4" -o "$CALLSIGN" = "E4" -o "$CALLSIGN" = "E4+1" -o "$CALLSIGN" = "Film4" -o "$CALLSIGN" = "ITV1" -o "$CALLSIGN" = "ITV1 +1" -o "$CALLSIGN" = "ITV2" -o "$CALLSIGN" = "ITV2 +1" -o "$CALLSIGN" = "ITV3" -o "$CALLSIGN" = "ITV4" -o "$CALLSIGN" = "Dave" -o "$CALLSIGN" = "Dave ja vu" ]; then
#	if [ "$CALLSIGN" = "FIVE USA" -o "$CALLSIGN" = "FIVER" -o "$CALLSIGN" = "FIVE" -o "$CALLSIGN" = "Channel 4" -o "$CALLSIGN" = "Channel 4+1" -o "$CALLSIGN" = "More 4" -o "$CALLSIGN" = "E4" -o "$CALLSIGN" = "E4+1" -o "$CALLSIGN" = "Film4" -o "$CALLSIGN" = "ITV1" -o "$CALLSIGN" = "ITV1 +1" -o "$CALLSIGN" = "ITV2" -o "$CALLSIGN" = "ITV2 +1" -o "$CALLSIGN" = "ITV3" -o "$CALLSIGN" = "ITV4" -o "$CALLSIGN" = "Dave" -o "$CALLSIGN" = "Dave ja vu" ]; then	
# Fails on FIVER and cuts almost entire recording. Works for other FIVE channels with caveat that they include news bulletins which are't cut
	if [ "$CALLSIGN" = "FIVE USA" -o "$CALLSIGN" = "FIVE" -o "$CALLSIGN" = "Channel 4" -o "$CALLSIGN" = "Channel 4+1" -o "$CALLSIGN" = "More 4" -o "$CALLSIGN" = "E4" -o "$CALLSIGN" = "E4+1" -o "$CALLSIGN" = "Film4" -o "$CALLSIGN" = "ITV1" -o "$CALLSIGN" = "ITV1 +1" -o "$CALLSIGN" = "ITV2" -o "$CALLSIGN" = "ITV2 +1" -o "$CALLSIGN" = "ITV3" -o "$CALLSIGN" = "ITV4" -o "$CALLSIGN" = "Dave" -o "$CALLSIGN" = "Dave ja vu" ]; then	
		echo >>$LOGFILE "Callsign in whitelist - will run silence_detect"		
		mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "update recorded set commflagged=2 where chanid=$CHANID and starttime='${STARTTIME}';" $MYTHDB
		CUTLIST=""
		echo >>$LOGFILE "silence_detect $FILENAME"
		silence_detect $FILENAME
		echo >>$LOGFILE "silect_detect() set CUTLIST to $CUTLIST"
		let BREAKS=`echo "$CUTLIST"|wc -l`/2
		echo >>$LOGFILE "$BREAKS break(s) found."
#		mythcommflag --setcutlist $CUTLIST -f $FILENAME
		mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "$CUTLIST" $MYTHDB
		RC=$?
#		echo "mythcommflag --setcutlist returned $RC"
		echo >>$LOGFILE "mysql setskiplist returned $RC"
		if [ $RC -eq 0 ]; then
			mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "update recorded set commflagged=1 where chanid=$CHANID and starttime='${STARTTIME}';" ${MYTHDB}
			mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "update jobqueue set status=272, comment='Finished, $BREAKS break(s) found.' where id=$JOB;" ${MYTHDB}			
			if [ $COPYTOCUTLIST -eq 1 ]; then
				mythcommflag --gencutlist -f $FILENAME
				RC=$?
				if [ $RC -eq 0 ]; then
					echo >>$LOGFILE "mythcommflag --gencutlist successfully copied skip list to cut list"
				else
					echo >>$LOGFILE "mythcommflag --gencutlist failed, returned $RC"		
				fi
			fi			
		else
			echo >>$LOGFILE "mythcommflag failed; returned $RC"
			mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "update recorded set commflagged=0 where chanid=$CHANID and starttime='${STARTTIME}';" ${MYTHDB}
		fi		
	else
		# Not a whitelisted channel for silence_detect
		echo >>$LOGFILE "won't run silence-detect, running mythcommflag $*"
		exec mythcommflag $*
		exit $?
	fi		
fi
echo >>$LOGFILE



Version using mythutil for 0.25+ and it no longer needs mythexport


Script.png mythcommflag-wrapper.sh

#!/bin/bash

# Version using mythutil for 0.25+ as it no longer needs mythexport: Tuned detect slightly, updated whitelist to include Freesat/Freeview +1 channels. Supports UK Freeview/Freesat HD Channels as they have MPEG audio channels. ( Channels with AC3 Only are not supported. ) Changed ffmpeg to avconv, as ffmpeg is deprecated. and ffmpeg -i occasionally chose the Audio Descriptive track, which is nearly all silence.
 
# edit/tune these #
# Allow ad breaks to be upto 400s long by coalescing non-silence
MAXCOMMBREAKSECS=400
# -80dB and minimum of 0.12s to be considered 'silent'
#MP3SPLT_OPTS="th=-70,min=0.15"
MP3SPLT_OPTS="th=-80,min=0.12"
# Log file
LOGFILE="/var/log/mythtv/mythcommflag-wrapper"
# Copy commercial skiplist to cutlist automatically? 1=Enabled
#COPYTOCUTLIST=1

MYTHCFG=""
# mythbackend in Fedora packages has $HOME=/etc/mythtv
if [ -e $HOME/mysql.txt ]; then
	MYTHCFG="$HOME/mysql.txt"
fi


if [ -e $HOME/.mythtv/mysql.txt ]; then
	MYTHCFG="$HOME/.mythtv/mysql.txt"
fi
if [ "$MYTHCFG" = "" ]; then
	echo >>$LOGFILE "No mysql.txt found in $HOME or $HOME/.mythtv - exiting!"
	exit 1
fi

# DB username and password
MYTHHOST=`grep DBHostName <$MYTHCFG | awk -F= '{print $2}'`
MYTHUSER=`grep DBUserName <$MYTHCFG | awk -F= '{print $2}'`
MYTHPASS=`grep DBPassword <$MYTHCFG | awk -F= '{print $2}'`
MYTHDB=`grep DBName <$MYTHCFG | awk -F= '{print $2}'`
#echo >>$LOGFILE "[$MYTHHOST] [$MYTHUSER] [$MYTHPASS] [$MYTHDB]"

# root of MythTV recordings
RECORDINGSROOT=`mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "select dirname from storagegroup;" $MYTHDB | tail -n +2 | sort | uniq | tr '\n' ' '`


echo >>$LOGFILE "$0 run with [$*] at `date` by `whoami`"
echo >>$LOGFILE "RECORDINGSROOT=$RECORDINGSROOT"

# From <http://www.mythtv.org/wiki/Silence-detect.sh>

silence_detect() {
		local filename=$1
		
		TMPDIR=`mktemp -d /tmp/mythcommflag.XXXXXX` || exit 1		

		cd $TMPDIR
		touch `basename $filename`.touch
		ionice -c3 nice avconv -i $filename -acodec copy sound.mp2
		ionice -c3 nice mp3splt -s -p $MP3SPLT_OPTS sound.mp2

		CUTLIST=`tail --lines=+3 mp3splt.log|sort -g |\
		       awk 'BEGIN{start=0;ORS=","}{if($2-start<'$MAXCOMMBREAKSECS')
		       {finish=$2} else {print int(start*25+1)"-"int(finish*25-25);
		       start=$1; finish=$2;}}END{print int(start*25+1)"-"9999999}'`
		echo >>$LOGFILE "silence-detect has generated cutlist: $CUTLIST"

		echo >>$LOGFILE "silence-detect(): CHANID=$CHANID, STARTTIME=$STARTTIME"		
		
		rm -rf $TMPDIR
}		


if [ $# -eq 0 ]; then
	# run with no parameters, flag every unflagged recording
	exec mythcommflag
	exit $?
else
	if [ $# -eq 4 -a "$1" = "-j" -a "$3" = "-V" ]; then
		# this is a manual flag job
		# TODO AJB20110129 generate a filelist from recorded table where commflagged=0 and cutlist=0
		# then run silence_detect over each filename
		JOB=$2
	else
		# we're being used in some other way, run the real mythcommmflag
		echo >>$LOGFILE "running mythcommflag $*"
		exec mythcommflag $*
		exit $?
	fi
	echo >>$LOGFILE "running job $JOB"
	HASCUTLIST=`mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "select recorded.cutlist from recorded join jobqueue where jobqueue.id=$JOB and jobqueue.chanid=recorded.chanid and jobqueue.starttime=recorded.starttime;" $MYTHDB | tail -n +2`	
	if [ "$HASCUTLIST" = "1" ]; then
		echo >>$LOGFILE "program already has (manual?) cutlist, exiting"
		exit 0
	fi
	CALLSIGN=`mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "select channel.callsign from channel join jobqueue where jobqueue.id=$JOB and jobqueue.chanid=channel.chanid;" $MYTHDB | tail -n +2`
	CHANID=`mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "select chanid from jobqueue where jobqueue.id=$JOB;" $MYTHDB | tail -n +2`	
	STARTTIME=`mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "select starttime from jobqueue where jobqueue.id=$JOB;" $MYTHDB | tail -n +2`
	echo >>$LOGFILE "channel callsign is $CALLSIGN"
	echo >>$LOGFILE "chanid=$CHANID STARTTIME=$STARTTIME"
	BASENAME=`mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "select recorded.basename from recorded join jobqueue where jobqueue.id=$JOB and jobqueue.chanid=recorded.chanid and jobqueue.starttime=recorded.starttime;" $MYTHDB | tail -n +2`	
	echo >>$LOGFILE "basename is $BASENAME"
	FILENAME=`ionice -c3 nice find ${RECORDINGSROOT} -name $BASENAME`
	echo >>$LOGFILE "filename is $FILENAME"	

        # See discussion page for proposed changes to this next section

	# make lower case and remove white space to account for minor 
	# presentation changes between transmissions and over time callsigns
	# on Freesat and freeview differ e.g. "FIVE *" and "Five*". Also 
	# removed is the +1 suffix from the criteria, if the primary channel
	# is allowable the +1 variant should also be included in white list.
	lowerCALLSIGN="$(echo $CALLSIGN | tr '[A-Z]' '[a-z]') | tr -d '+1' | tr -d '[:space:]'"

	case $lowerCALLSIGN in # the following list should be in lower case and without any white space and no need to specify "+1" variants
	# E4/channel 4 treats trailers as program. The silence markers are 
	# only either side of actual adverts. E4 often has trailers before and
	# after adverts, which will not be cut, and appear within the wanted show. 
	# Works for other FIVE channels with caveat that they include news 
	# bulletins which are't cut
	"five"|"5"|"fiveusa"|"5usa"|"channel4"|"channel4hd"|"more4"|"e4"|"film4"|"4seven"|"itv1"|"itv1hd"|"itv2"|"itv3"|"itv4"|"dave"|"davejavu")
		echo >>$LOGFILE "Callsign in whitelist - will run silence_detect"		
		mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "update recorded set commflagged=2 where chanid=$CHANID and starttime='${STARTTIME}';" $MYTHDB
		CUTLIST=""
		echo >>$LOGFILE "silence_detect $FILENAME"
		silence_detect $FILENAME
		echo >>$LOGFILE "silence_detect() set CUTLIST to $CUTLIST"
		let BREAKS=`grep -o "-" <<< "$CUTLIST" | wc -l`
		echo >>$LOGFILE "$BREAKS break(s) found."
                mythutil --setskiplist $CUTLIST --chanid="$CHANID" --starttime="${STARTTIME}"
		RC=$?
		echo >>$LOGFILE "mythutil setskiplist returned $RC"
		if [ $RC -eq 0 ]; then
			mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "update recorded set commflagged=1 where chanid=$CHANID and starttime='${STARTTIME}';" ${MYTHDB}
			mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "update jobqueue set status=272, comment='Finished, $BREAKS break(s) found.' where id=$JOB;" ${MYTHDB}			
			if [ $COPYTOCUTLIST -eq 1 ]; then
				mythutil --gencutlist --chanid="$CHANID" --starttime="${STARTTIME}"
				RC=$?
				if [ $RC -eq 0 ]; then
					echo >>$LOGFILE "mythutil --gencutlist successfully copied skip list to cut list"
				else
					echo >>$LOGFILE "mythutil --gencutlist failed, returned $RC"		
				fi
			fi			
		else
			echo >>$LOGFILE "mythcommflag failed; returned $RC"
			mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "update recorded set commflagged=0 where chanid=$CHANID and starttime='${STARTTIME}';" ${MYTHDB}
		fi
		;;		
	*)
		# Not a whitelisted channel for silence_detect
		echo >>$LOGFILE "won't run silence-detect, running mythcommflag $*"
		exec mythcommflag $*
		exit $?
		;;
	esac		
fi
echo >>$LOGFILE

Info from trac Ticket #11936:-


In the later version of mythcommflag-wrapper.sh (ver 0.25+) there is an  error in the lowerCALLSIGN. The ')' is too early and should be before the  final '"'

I'm no coder. I couldn't possibly comment. Stevegoodey 11:40, 12 November 2013 (UTC)

Edit script variables

Edit the environment variables in the wrapper script according to your local installation.

Enable commflagging

Check that commflagging is allowed in mythtv-setup (i.e. the JobAllowCommFlag setting should be 1).

Test

In the 'Watch Recordings' browser, select a recording that was from a channel included in the script's whitelist (Film4, Dave, Dave ja vu, ITV1-4, ITV1 +1, ITV2 +1), hit menu, then Jobs, then 'Begin Advert Detection'). Within a few minutes, that recording should be commflagged appropriately.

Future Planned Improvements

Fill in the TODOs. Done, apart from automatic "flag all unflagged recordings" mode. Maybe when it can be trusted to not go haywire...

Unnecessary. Running mythcommflag --queue will queue all unflagged recordings for commercial flagging in the jobqueue.

Expand the channel whitelist ("Five" tested once and failed, others???)

More sanity checks of mysql client exit codes and parsed variables.

Refine $CUTLIST generation so that we print $start instead of ($finish*25-25) if ($finish*25-25) < ($finish*25+1).

Per-channel profiles instead of whitelisting.

Experiment with other mp3splt options for other channels. Contributions here particularly welcome!

Merge this into the real mythcommflag. Will probably need a new column in the channels table to signify which commflagging method should be used, and that in turn will require some work to the channel-editor UI of mythtv-setup.

For 0.27+. Change to blacklisting and using HD

By ceesquared. Many thanks to cowbutt for starting this.

This version changes the the script to use blacklisting instead of whitelisting on the assumption that anything transmitted on Freesat/freeview will conform to the silence break requirement. It also incorporates the changes in 0.27 that removes mysql.txt and replaces it with config.xml. If you find a channel that does not conform then you can add it to the blacklist after examining the log output. It has been checked on 0.28.


Script.png mythcommflag-wrapper.sh

#!/bin/bash

# place in /usr/bin/
# Version using mythutil for 0.25+ as it no longer needs mythexport: Only checked on Freesat HD but it should work with 
# Freeview HD if they have AC3 audio.  Changed ffmpeg to avconv, as ffmpeg is deprecated. avconv used the Audio Descriptive
# track without the ac3 adjustment. On my test this was audio+ AD but I am told that it could be nearly all silence so I 
# forced the use of the ac3 track.
# Edited version which assumes all freesat/freeview channels conform to the silent gap requirement. There is therefore no 
# need for a whitelist.
# Updated for 0.27 as mysql.txt no longer exists.

# Function to extract data from config.xml

Extract(){
Sourcefile=$1
Searchterm=$2
# $1 is filename, $2 is search item
Filterterm="<"$Searchterm">"
Result=`grep $Filterterm <$Sourcefile `
Result=`echo ${Result//$Filterterm/}`
Filterterm="</"$Searchterm">"
Result=`echo ${Result//$Filterterm/}`
echo $Result
}
#End Extract

# From <http://www.mythtv.org/wiki/Silence-detect.sh>

silence_detect() {
		local filename=$1
		
		TMPDIR=`mktemp -d /tmp/mythcommflag.XXXXXX` || exit 1		

		cd $TMPDIR
		touch `basename $filename`.touch
# Find out if ac3(HD)
	avprobe -show_streams -select_streams a $filename | grep ac3 >/dev/null
	if [ $? -eq 0 ];then # ac3 stream
# use avconv to get stream location from stderr/stdout. Info is in stderr so it has to be piped to stdout to see it(2>&1).
# find the line with ac3 This has the format     Stream #m:n[0xhex]...............
	streamline=$(avconv -hide_banner -i $filename 2>&1 | grep ac3)
# Now strip that line to leave m:n
# strip off leading characters
	stream1=${streamline#*#} # leaving m:n[0xhex]........
# we now strip the trailing characters 
	stream2=${stream1%[0x*} # gives m:n in stream2
# export ac3 audio in mp2 format
	ionice -c 3 nice avconv -i $filename -map $stream2 -codec:a mp2 sound.mp2
	else ionice -c 3 nice avconv -i $filename -acodec copy sound.mp2
	fi
	ionice -c 3 nice mp3splt -s -p $MP3SPLT_OPTS sound.mp2

	CUTLIST=`tail --lines=+3 mp3splt.log|sort -g |\
	       awk 'BEGIN{start=0;ORS=","}{if($2-start<'$MAXCOMMBREAKSECS')
	       {finish=$2} else {print int(start*25+1)"-"int(finish*25-25);
	       start=$1; finish=$2;}}END{print int(start*25+1)"-"9999999}'`
	echo "silence-detect has generated cutlist: $CUTLIST" >>$LOGFILE 

	echo "silence-detect(): CHANID=$CHANID, STARTTIME=$STARTTIME" >>$LOGFILE
# remove the temporary directory and its contents		
	rm -rf $TMPDIR
}
# End Silence detect		

# Main body
 
# edit/tune these #
# Allow ad breaks to be up to 400s long by coalescing non-silence
MAXCOMMBREAKSECS=400
# -80dB and minimum of 01.0s to be considered 'silent'
# Limited investigation has shown that the advert gap is about 2 secs and the between adverts
# gap about 1 sec
#MP3SPLT_OPTS="th=-70,min=0.15"
MP3SPLT_OPTS="th=-80,min=1.0"
#MP3SPLT_OPTS="th=-80,min=0.50"
#MP3SPLT_OPTS="th=-60,min=0.10"
# Log file
LOGFILE="/var/log/mythtv/mythcommflag.log"
# Save passed parameters
NOPARAM=$#
PARAMSTR=$*
if [ $NOPARAM -eq 4 -a "$1" = "-j" -a "$3" = "-v" ]; then
		# this is a manual flag job
		JOB=$2
		MANFLAG=true
fi

echo "starting Run ukcommflag $date" >>$LOGFILE

# Copy commercial skiplist to cutlist automatically? 1=Enabled
#COPYTOCUTLIST=1

MYTHCFG=""
# Find config.xml in Ver 0.27 as mysql.txt no longer exists
if [ -e $HOME/.mythtv/config.xml ]; then
	MYTHCFG="$HOME/.mythtv/config.xml"
fi


if [ -e $MYTHCONFDIR/config.xml ]; then
	MYTHCFG="$MYTHCONFDIR/config.xml"
fi
if [ "$MYTHCFG" = "" ]; then
	echo "No config.xml found in $MYTHCONFDIR or $HOME/.mythtv - exiting!" >>$LOGFILE
	exit 1
fi

# DB username and password
MYTHHOST=$(Extract $MYTHCFG "Host")
MYTHUSER=$(Extract $MYTHCFG "UserName")
MYTHPASS=$(Extract $MYTHCFG "Password")
MYTHDB=$(Extract $MYTHCFG "DatabaseName")
# echo "Host, Username, Password, and Database name extracted" >>$LOGFILE
# echo $MYTHHOST"   "$MYTHUSER"   "$MYTHPASS"   "$MYTHDB >>$LOGFILE

# root of MythTV recordings
RECORDINGSROOT=`mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "select dirname from storagegroup;" $MYTHDB | tail -n +2 | sort | uniq | tr '\n' ' '`


echo "$0 run with [$PARAMSTR] at `date` by `whoami`" >>$LOGFILE
echo "RECORDINGSROOT=$RECORDINGSROOT" >>$LOGFILE


if [ NOPARAM -eq 0 ]; then
	# run with no parameters, flag every unflagged recording
	exec mythcommflag
	exit $?
else
	if [ MANFLAG -ne true ]; then
	# we're being used in some other way, run the real mythcommmflag
		exec mythcommflag $PARAMSTR
		exit $?
fi
echo "running job $JOB" >>$LOGFILE
HASCUTLIST=`mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "select recorded.cutlist from recorded join jobqueue where jobqueue.id=$JOB and jobqueue.chanid=recorded.chanid and jobqueue.starttime=recorded.starttime;" $MYTHDB | tail -n +2`	
if [ "$HASCUTLIST" = "1" ]; then
	echo "program already has (manual?) cutlist, exiting" >>$LOGFILE
	exit 0
fi
CALLSIGN=`mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "select channel.callsign from channel join jobqueue where jobqueue.id=$JOB and jobqueue.chanid=channel.chanid;" $MYTHDB | tail -n +2`
CHANID=`mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "select chanid from jobqueue where jobqueue.id=$JOB;" $MYTHDB | tail -n +2`	
STARTTIME=`mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "select starttime from jobqueue where jobqueue.id=$JOB;" $MYTHDB | tail -n +2`
echo "channel callsign is $CALLSIGN" >>$LOGFILE
echo "chanid=$CHANID STARTTIME=$STARTTIME" >>$LOGFILE
BASENAME=`mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "select recorded.basename from recorded join jobqueue where jobqueue.id=$JOB and jobqueue.chanid=recorded.chanid and jobqueue.starttime=recorded.starttime;" $MYTHDB | tail -n +2`	
echo "basename is $BASENAME" >>$LOGFILE
FILENAME=`ionice -c 3 nice find ${RECORDINGSROOT} -name $BASENAME`
echo "filename is $FILENAME" >>$LOGFILE


	# make lower case and remove white space to account for minor 
	# presentation changes between transmissions and over time callsigns
	# on Freesat and freeview differ e.g. "FIVE *" and "Five*". Also 
	# removed is the +1 suffix from the criteria, if the primary channel
	# is allowable the +1 variant should also be included in white list.
        # Remove HD on the same logic as +1
        # Convert to lower case
lowerCALLSIGN="$(echo $CALLSIGN | tr '[A-Z]' '[a-z]')"
        # Remove +1
lowerCALLSIGN=`echo ${lowerCALLSIGN//+1/}`
	# Remove +24
lowerCALLSIGN=`echo ${lowerCALLSIGN//+24/}`
	#Remove hd
lowerCALLSIGN=`echo ${lowerCALLSIGN//hd/}`
	#Remove spaces
lowerCALLSIGN="$(echo $lowerCALLSIGN | tr -d '[:space:]')"
     case $lowerCALLSIGN in
# the following list should be in lower case and without any white space and no need to specify "+1" or HD variants
	"xxx"|"yyy")
	# A Blacklisted channel for silence_detect
	echo "won't run silence-detect on $CALLSIGN, running mythcommflag $PARAMSTR" >>$LOGFILE
	exec mythcommflag $PARAMSTR
	exit $?
	;;
	*)
	# E4/channel 4 treats trailers as program. The silence markers are 
	# only either side of actual adverts. E4 often has trailers before and
	# after adverts, which will not be cut, and appear within the wanted show. 
	# Works for other FIVE channels with caveat that they include news 
	# bulletins which are't cut

        echo "Callsign $CALLSIGN not in Blacklist - will run silence_detect" >>$LOGFILE
        mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "update recorded set commflagged=2 where chanid=$CHANID and starttime='${STARTTIME}';" $MYTHDB
        CUTLIST=""
        silence_detect $FILENAME
        echo "silence_detect() set CUTLIST to $CUTLIST" >>$LOGFILE
        let BREAKS=`grep -o "-" <<< "$CUTLIST" | wc -l`
        echo "$BREAKS break(s) found." >>$LOGFILE
        mythutil --setskiplist $CUTLIST --chanid="$CHANID" --starttime="${STARTTIME}"
        RC=$?
        echo "mythutil setskiplist returned $RC" >>$LOGFILE
        if [ $RC -eq 0 ]; then
	     mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "update recorded set commflagged=1 where chanid=$CHANID and starttime='${STARTTIME}';" ${MYTHDB}
        	mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "update jobqueue set status=272, comment='Finished, $BREAKS break(s) found.' where id=$JOB;" ${MYTHDB}			
	     if [ $COPYTOCUTLIST -eq 1 ]; then
		mythutil --gencutlist --chanid="$CHANID" --starttime="${STARTTIME}"
		RC=$?
		if [ $RC -eq 0 ]; then
			echo "mythutil --gencutlist successfully copied skip list to cut list" >>$LOGFILE
		else
			echo "mythutil --gencutlist failed, returned $RC" >>$LOGFILE
		fi
	     fi			
        else
	echo "mythcommflag failed; returned $RC" >>$LOGFILE
	mysql -h${MYTHHOST} -u${MYTHUSER} -p${MYTHPASS} -e "update recorded set commflagged=0 where chanid=$CHANID and starttime='${STARTTIME}';" ${MYTHDB}
        fi
        ;;		
     esac		
fi
echo "Run Complete" >>$LOGFILE

If you enable logging then make sure that you have write permission for the logfile.