Perl script: Spliting and merging huge files

split-file.pl

#!/usr/bin/perl
use warnings;
use strict;

use Getopt::Long;

my $chunk_size = 1024*1024;
my $buf_size   =   10*1024;
my $dest_dir   = '.';
GetOptions (
  'buf-size:i'    => \$buf_size,
  'chunk-size:i'  => \$chunk_size,
  'dest-dir:s'    => \$dest_dir
) or die;

my $filename = shift or die "No filename given";

my ($filename_base) = $filename =~ m!([^/]+)$!;

die "Dest dir $dest_dir is no directory" unless -d $dest_dir;

open (my $in, '<', $filename) or die "Could not open $filename\n$!";
binmode $in;

my $cnt_chunks = 0;
my $cnt_bufs   = 0;

my $out = open_out_part($cnt_chunks);

while (read($in, my $buf, ($cnt_bufs+1) * $buf_size < $chunk_size ?  $buf_size : ($cnt_bufs+1)*$buf_size - $chunk_size)) {

    $cnt_bufs ++;
    print $out $buf;

    if ($cnt_bufs * $buf_size > $chunk_size) {
       $cnt_chunks ++;
       $cnt_bufs = 0;
       close $out;

       $out = open_out_part($cnt_chunks);
    }
}

close $out;
close $in;

sub open_out_part { #_{
  my $cnt = shift;
  my $filename = sprintf("$dest_dir/$filename_base.%03d", $cnt);

  open (my $out, '>', $filename) or die "Could not open $filename\n$!";

  binmode $out;

  return $out;
} #_}

merge-file.pl

#!/usr/bin/perl
use warnings;
use strict;

my $filename = shift or die;

die unless -e "$filename.000";

my $cnt_in = 0;
open (my $out, '>', $filename) or die;
binmode $out;

while (-e sprintf("$filename.%03d", $cnt_in)) {

  open (my $in, '<', sprintf("$filename.%03d", $cnt_in)) or die;
  binmode $in;
  print "$cnt_in\n";
  while (read($in, my $buf, 10000)) {
    print $out $buf;
  }
  close $in;

  $cnt_in++;
}
close $out;

See also