#!/usr/bin/perl -w

use strict; use warnings; 


my $base_url  = 'http://www.sec.gov/Archives';
my @aonly=qw{
edgar/data/1122304/0001193125-15-118890.txt'
edgar/data/1640984/0001052918-16-000754.txt         
edgar/data/1640984/0001052918-16-000803.txt         
edgar/data/1084869/0001437749-16-024828.txt         
edgar/data/1084869/0001084869-16-000045.txt         
edgar/data/1084869/0001084869-16-000046.txt         
edgar/data/1084869/0001084869-16-000047.txt         
edgar/data/1084869/0000950162-16-000085.txt         
edgar/data/1511144/0001206774-16-004470.txt         
edgar/data/1665022/0001665022-16-000001.txt         
edgar/data/1662965/0001662965-16-000001.txt         
edgar/data/1651654/0001651654-16-000002.txt         
edgar/data/1664237/0001664237-16-000001.txt         
edgar/data/1664513/0001664513-16-000001.txt         
edgar/data/1665711/0001665711-16-000001.txt         
edgar/data/1665354/0001665354-16-000001.txt         
edgar/data/1664635/0001664635-16-000001.txt         
edgar/data/1625109/0001625109-16-000002.txt         
edgar/data/1658659/0001658659-16-000002.txt         
edgar/data/1666635/0001666635-16-000001.txt         
edgar/data/1614102/0001614102-16-000002.txt         
edgar/data/1665218/0001665218-16-000001.txt         
edgar/data/1663921/0000905729-16-000383.txt         
edgar/data/1666561/0001666561-16-000002.txt         
edgar/data/1668972/0001668972-16-000002.txt         
edgar/data/1540531/0000905718-16-001186.txt         
edgar/data/1540531/0000905718-16-001254.txt         

            }; 

my $file_count=0;     
my $FH_OUT=\*STDOUT; 
my @fields=qw/cik form_type report_date file_date name/; 
foreach my $filetoget(@aonly) { 
  my $res=get_process_trunc ($filetoget); 
  if (scalar(keys(%$res))) { 
      my $lineout=''; 
      for my $field (@fields){ 
        if ($res->{$field}) {$lineout.=$res->{$field}} 
        $lineout.='|';
        } 
      print $FH_OUT $lineout."\n"; 
      } 
  } 

exit; 

sub get_process_trunc {    
# http://www.perlmonks.org/?node_id=1183107
  my $filetoget=shift; 
  my $fullfile="$base_url/$filetoget";
  my $res={}; 

  use LWP::UserAgent;

  my $received_size = 0;
  my $partial = '';
  my $ua = LWP::UserAgent->new;
  my $response = $ua->get($fullfile
                         , ':content_cb'=> sub { 
                               my ($data, $response, $protocol) = @_;
                               $partial.=$data; 
                               $received_size += length $data;                      
                               die if ($received_size>10000); 
                               # die inside this callback interrupt the request, not the program!!
                               } 
                         );
  if ($partial) {
#      print 'length:'.length($partial)."\n";
      my $line_count=0; 
      for my $line (split qr/\'\n'/, $partial) { 
        if($line=~m/^\s*CENTRAL\s*INDEX\s*KEY:\s*(\d*)/m)             {$res->{cik}        =$1;}
        if($line=~m/^\s*FORM\s*TYPE:\s*(.*$)/m)                       {$res->{form_type}  =$1;}
        if($line=~m/^\s*CONFORMED\s*PERIOD\s*OF\s*REPORT:\s*(\d*)/m)  {$res->{report_date}=$1;}
        if($line=~m/^\s*FILED\s*AS\s*OF\s*DATE:\s*(\d*)/m)            {$res->{file_date}  =$1;}
        if($line=~m/^\s*COMPANY\s*CONFORMED\s*NAME:\s*(.*$)/m)        {$res->{name}       =$1;}
        $line_count++;
        last if ($line_count>50); 
        } 
      } # success 
  return $res; 

  } # get_process_trunc