rclone upload/download split/restore large file
Last update
2020-03-21
2020-03-21
«upload/backup a big file to your online drive with auto split/concatenation»
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 | #!/usr/bin/env ruby require 'shellwords' def print_usage_and_exit progr = File.basename __FILE__ puts "USO: #{progr} <service_name> <ul|dl|rm|ls> <src_file|-> [<dst_file|->] [chunk_size_MB]" puts " splits/concatenate and upload/download a file via rclone" puts " eg: tar -cf - / | pigz | #{progr} gdrive ul - /backup/vps/root.tgz" puts " eg: #{progr} gdrive dl /backup/vps/root.tgz - | tar -xzf - -C /" puts " eg: #{progr} gdrive ul homes.tgz /backup/vps/homes.tgz" puts " eg: #{progr} gdrive dl /backup/vps/homes.tgz homes.tgz" puts " eg: #{progr} gdrive rm /backup/vps/homes.tgz" puts " eg: #{progr} gdrive ls /backup/vps" exit 1 end # temp files cleanup on exit $temp_files = [] def remove_temp_files; $temp_files.each{|f| File.unlink(f) rescue nil }; end Signal.trap('INT' ){ remove_temp_files; exit } at_exit{ remove_temp_files } # sanitize parameters srv_name, action, src_file, dst_file, chunk_size = ARGV srv_name = srv_name.to_s.strip src_file = src_file.to_s.strip dst_file = dst_file.to_s.strip chunk_size = 1024 * 1024 * (chunk_size.to_i > 0 ? chunk_size.to_i : 100) # default 100MB case action when 'rm' print_usage_and_exit if ARGV.size < 3 puts "deleting files..." system %Q|rclone --include #{src_file.shellescape}.rc\\* delete #{srv_name}:| when 'ls' print_usage_and_exit if ARGV.size < 3 puts "listing files..." files = `rclone lsf #{srv_name}:#{src_file.shellescape}`.split("\n") puts files.map{|i| i =~ /\.rc[0-9]{4}/ ? "*#{i[0..-8]}" : " #{i}"}.sort.uniq when 'ul' print_usage_and_exit if ARGV.size < 4 puts "deleting any eventual old files..." system %Q|rclone --include #{dst_file.shellescape}.rc\\* delete #{srv_name}:| puts "split and upload of [#{src_file}]:" fd_src = src_file == '-' ? STDIN : File.open(src_file) chunk_idx = -1 until fd_src.eof? remote_name = "#{dst_file}.rc#{"%04d" % (chunk_idx+=1)}" tmp_name = `mktemp -u /tmp/rclone-bkup.XXXXXXXXXX`.strip $temp_files << tmp_name File.open(tmp_name, 'wb'){|f| f << fd_src.read(chunk_size) } ObjectSpace.garbage_collect puts " #{tmp_name} => #{remote_name}" system %Q|rclone copyto #{tmp_name.shellescape} #{srv_name}:#{remote_name.shellescape}| File.unlink tmp_name end fd_src.close when 'dl' print_usage_and_exit if ARGV.size < 4 remote_names = `rclone --include #{src_file.shellescape}.rc\\* lsf #{srv_name}: -R --files-only`.split("\n").sort if remote_names.size == 0 puts "no file found" exit 1 end STDERR.puts "download and restore to [#{dst_file}]:" fd_dst = dst_file == '-' ? STDOUT : File.open(dst_file, 'wb') remote_names.each do |remote_name| tmp_name = `mktemp -u /tmp/rclone-bkup.XXXXXXXXXX`.strip $temp_files << tmp_name STDERR.puts " #{remote_name} => #{tmp_name}" system %Q|rclone copyto #{srv_name}:#{remote_name.shellescape} #{tmp_name.shellescape}| File.open(tmp_name) do |fd_chunk| # apppendi il pezzo 10MB al colpo fd_dst << fd_chunk.read(1024 * 1024 * 10) until fd_chunk.eof? end ObjectSpace.garbage_collect File.unlink tmp_name end fd_dst.close else puts "action [#{action}] unknown" print_usage_and_exit end |