From 6b02f95eb097c67ba9c3eeef0453c911e5133924 Mon Sep 17 00:00:00 2001 From: Michael Reber Date: Wed, 4 Mar 2020 23:12:54 +0100 Subject: [PATCH] Initial commit --- Gemfile | 3 + README.md | 56 ++++++++++++- config.yml | 6 ++ convertVideos.rb | 204 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 267 insertions(+), 2 deletions(-) create mode 100644 Gemfile create mode 100644 config.yml create mode 100644 convertVideos.rb diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..b5cb8f8 --- /dev/null +++ b/Gemfile @@ -0,0 +1,3 @@ +source 'https://rubygems.org' +gem 'streamio-ffmpeg' +gem 'peach' \ No newline at end of file diff --git a/README.md b/README.md index 88d0e2d..5f3f738 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,55 @@ -# video-in-place-hevc-converter +# Convert videos in-place to h265 (HEVC) -This is a simple ruby script that will chuch through a directory converting all of the videos to use h.265 or HEVC in place. \ No newline at end of file +This is a simple ruby script that will chuch through a directory converting all of the videos to use h.265 or HEVC in place. This works by converting the video file to \*.tmp.mp4, and then moving it to its original file name with the mp4 extension. It will delete the old version of the file. + + +## Usage + +This is a long running ruby script, it makes calls to FFMPEG using a ruby gem to scrape metadata of videos, and transcode them. It works by appling some simple filters to create a list of videos that can be converted, and then works through that queue. + +Move to the directory where the config is located and run: `ruby convertVideos.rb start` + + +## Example Config +Please note the preceding colons are important. Also the file must be called config.yml + +``` +:min_age_days: 5 +:directory: /home/user/videos/movies +:log_location: /home/user/logs/HevcConversion.log +:preset: slow +:max_new_file_size_ratio: 0.9 +``` + + +|Configuration| Description| +|--|--| +|directory | The directory to recurse into. All files will be considered within that directory. | +|min_age_days | How many days old does this file have to be to be considered for conversion. | +|log_location| This Script is designed to run deetached in the background. As such the log location is the best way to figure out whatis going on and the status of the conversion | +| preset | used to trade off between final file size, quality, and transcoding time. I recomend slow. See ffmpeg docs for more detail. | +|threads | How many threads should be used for converting the file. | +|max_new_file_size_ratio| Once transocing an individual file is finished, this script will make sure the output is smaller than the origional. Spesifically new file size <= Old file size * this value. Since transcoding always involves quality loss this value should be less than 1.0 | + + +## Coridnation +When this script starts to convert a video, it creates a .filename.tmp.mp4 file that used the old files filename. This acts as kind of a lock because the exitsance of that file is checked before conversion. This also allows us to save state between runs without needing to share a database or other coridination servcie. That file is left behind if, for any reason, the conversion fails, the process is stopped, or if afterthe conversion the new HEVC file is not at least 10% smaller than the origional. The content is replace with an explination if possible. + +It also makes it so multiple computers can run this script, provided they are all run against the same backing file system (SMB, NFS, etc.). There is a risk of duplicate work if two processes try to start to transcode the same file at the same time, but this risk is minimal for large libraries. To further mitigate, the list of candidate files is randomaized. + + +## Setup + +1. Instal Ruby 2.1.0+ +2. Install (ffmpeg)[https://ffmpeg.org/download.html] near version 4.2.2 `sudo apt-get install ffmpeg` +3. gem install `streamio-ffmpeg` +4. Optional: Install screen or tmux. This is to allow it to run in the background after closing SSH on a server. +5. Edit the script. +6. Run the script. +7. Automate/cron? + + +## Disclaimers +- Only use with videos you have the rights to copy +- This will delete the original video, so use with care. Test with a test directory before running on your entire library. +- Use a test file with all your media playing devices to ensure that they can handle HEVC encoding. Raspberry pies, both 1 and 2 are not able to handle HEVC decoding. diff --git a/config.yml b/config.yml new file mode 100644 index 0000000..73b3716 --- /dev/null +++ b/config.yml @@ -0,0 +1,6 @@ +:min_age_days: 0 +:directory: /mnt/movies +:log_location: ./log.txt +:preset: slower +:threads: 8 +:max_new_file_size_ratio: 0.9 \ No newline at end of file diff --git a/convertVideos.rb b/convertVideos.rb new file mode 100644 index 0000000..887bc03 --- /dev/null +++ b/convertVideos.rb @@ -0,0 +1,204 @@ +require 'rubygems' +require 'streamio-ffmpeg' +require 'fileutils' +require 'logger' +require 'yaml' +require 'peach' + +VID_FORMATS = %w[.avi .flv .mkv .mov .mp4] + +# Loads config file: +@config=YAML.load(File.read("./config.yml")) + +# Logger setup stuff: +@logger=Logger.new(@config[:log_location]) +@logger.level=Logger::INFO + +@logger.info "\n New Run starting now......" +@logger.info "Config is being used: #{@config}" + +# Check the file age: +def file_age(name) + (Time.now - File.ctime(name))/(24*3600) +end + + +def seconds_to_s(total_time) + total_time=total_time.to_int + return [total_time / 3600, total_time/ 60 % 60, total_time % 60].map { |t| t.to_s.rjust(2,'0') }.join(':') +end + +# Creates a list (array) of all aged files: +def get_aged_files(directory) + out=[] + Dir.foreach(directory){|file| + next if file == '.' or file == '..' or file.start_with?('.') + fileName=File.join(directory,file) + if(File.file?(fileName)) then + if VID_FORMATS.include? File.extname(file) then + if(file_age(fileName)>=@config[:min_age_days]) then + out< e + @logger.error "Problem processing a video" + @logger.error e + end + return nil +end + +# Main function for conversion: +def convert_file(original_video,filename) + options={ + video_codec: 'libx265', + threads: @config[:threads], + custom: "-preset #{@config[:preset]} -crf 25 -c:a copy".split + } + outFileName = get_base_name(filename) + error_thrown=nil + begin + startTime=Time.now + out = original_video.transcode(get_temp_filename(filename),options){ |progress| + duration=Time.now-startTime + remaining=(duration/progress)*(1-progress) + if(remaining>99999999) then + print "Progress converting #{filename.split('/').last} : #{(progress*100).round(1)}% \r" + else + print "Progress converting #{filename.split('/').last} : #{(progress*100).round(1)}% ETA is #{seconds_to_s(remaining)} \r" + end + } + rescue StandardError => e + error_thrown=e + puts e.to_s + end + puts "Done with #{filename.split('\\').last}" + if ( error_thrown ) + @logger.error "A video file failed to transcode correctly" + @logger.error error_thrown + FileUtils.rm(get_temp_filename(filename)) if File.exists?(get_temp_filename(filename)) + FileUtils.touch(get_temp_filename(filename)) + File.write(get_temp_filename(filename), + [ + 'An exception occured while transocding this movie.', + error_thrown + ].join('\n')) + elsif (out.size>original_video.size*@config[:max_new_file_size_ratio]) + @logger.warn "A video file, after transcoding was not at least #{@config[:max_new_file_size_ratio]} the size of the origional (new: #{out.size} old: #{original_video.size}). Keeping origonal #{filename}" + FileUtils.rm(get_temp_filename(filename)) + FileUtils.touch(get_temp_filename(filename)) + File.write(get_temp_filename(filename), "transcoded video not enough smaller than the origional.") + return nil + else + FileUtils.mv(get_temp_filename(filename),"#{outFileName}.mp4") + if filename!="#{outFileName}.mp4" then + FileUtils.rm(filename) + end + return out + end +end + +def status(app) + possible_files=get_aged_files(@config[:directory]) + puts "There are a total of #{possible_files.size} files that may need to be converted." + + candidate_files= get_candidate_files(possible_files) + + puts "There are a total of #{candidate_files[:movies].size} files that have not been converted yet." + puts "Total Duration: #{seconds_to_s(candidate_files[:runtime])}" +end + +@total_processing_time=0 +@processed_video_duration=0 + +def iterate + possible_files=get_aged_files(@config[:directory]) + @logger.info "There are a total of #{possible_files.size} files that may need to be converted." + + @logger.debug "Files to be checked: #{possible_files}" + + candidate_files= get_candidate_files(possible_files) + + @logger.info "There are a total of #{candidate_files[:movies].size} files that have not been converted yet." + @logger.debug "Candidate Files that need to be re-encoded: #{candidate_files}" + @logger.info "Total Duration: #{seconds_to_s(candidate_files[:runtime])}" + remaining_runtime=candidate_files[:runtime] + + + candidate_files[:movies].each_with_index do |file,index| + @logger.info "Starting to transcode file #{index+1} of #{candidate_files[:movies].size}: #{file}" + unless does_video_need_conversion?(file) + @logger.info "Video already converted, scanning again" + return true + end + startTime=Time.now + video=FFMPEG::Movie.new(file) + converted_video=safe_convert_file(video,file) + duration=Time.now - startTime + remaining_runtime-=video.duration + if !converted_video.nil? then + @total_processing_time+=duration + @processed_video_duration+=video.duration + end + avg=@processed_video_duration/@total_processing_time + @logger.info "Average videotime/walltime: #{avg} Estimated time remaining #{seconds_to_s(remaining_runtime/avg)}" + end + return false +end + +while iterate do end \ No newline at end of file