Beefy Boxes and Bandwidth Generously Provided by pair Networks
Keep It Simple, Stupid

Re^2: Help With Online Table Scraper

by Anonymous Monk
on Mar 03, 2011 at 09:40 UTC ( #891181=note: print w/replies, xml ) Need Help??

in reply to Re: Help With Online Table Scraper
in thread Help With Online Table Scraper

Sinistral is right, use a documented API whenever available, scraping is a fragile PITA :)
#!/usr/bin/perl -- use strict; use warnings; use Data::Dumper; use Web::Scraper; use File::Slurp qw' read_file '; use URI; Main( @ARGV ); exit( 0 ); sub Main { my $uri = URI->new(' +stics'); if(@_){ my $html_content = read_file( @_ ); ScrapePrint(\$html_content, $uri ); } else { ScrapePrint( $uri ); } } sub ScrapePrint { my $scraper = scraper { process q~//table[@id='yfncsumtab']/tr/td/table[8]//tr//tr~, 'Balance Sheet[]' => scraper { process '/tr/td[1]', 'key', 'TEXT'; process '/tr/td[2]', 'value', 'TEXT'; }, process #~ q~//table[@id='yfncsumtab']/tr/td/table[9]//tr//tr~, q~//table[@id='yfncsumtab']//table[9]//table//tr~, 'Cash Flow Statement[]' => scraper { process '/tr/td[1]', 'key', 'TEXT'; process '/tr/td[2]', 'value', 'TEXT'; }, process q~//table[@id='yfncsumtab']/tr[2]/td[3]/table[4]//tr//tr~, 'Dividends & Splits[]' => scraper { process '/tr/td[1]', 'key', 'TEXT'; process '/tr/td[2]', 'value', 'TEXT'; }, process q~//table[@id='yfncsumtab']/tr/td/table[4]//tr//tr~, 'Fiscal Year[]' => scraper { process '/tr/td[1]', 'key', 'TEXT'; process '/tr/td[2]', 'value', 'TEXT'; }, process q~//table[@id='yfncsumtab']/tr/td/table[7]//tr//tr~, 'Income Statement[]' => scraper { process '/tr/td[1]', 'key', 'TEXT'; process '//tr/td[2]', 'value.', 'TEXT'; }, process q~//table[@id='yfncsumtab']/tr/td/table[6]//tr//tr~, 'Management Effectiveness[]' => scraper { process '/tr/td[1]', 'key', 'TEXT'; process '/tr/td[2]', 'value', 'TEXT'; }, process q~//table[@id='yfncsumtab']/tr/td/table[5]//tr//tr~, 'Profitability[]' => scraper { process '/tr/td[1]', 'key', 'TEXT'; process '/tr/td[2]', 'value', 'TEXT'; }, process q~//table[@id='yfncsumtab']/tr/td[3]/table[3]//tr//tr~, 'Share Statistics[]' => scraper { process '/tr/td[1]', 'key', 'TEXT'; process '/tr/td[2]', 'value', 'TEXT'; }, process q~//table[@id='yfncsumtab']/tr[2]/td[3]/table[2]/tr//tr~, 'Stock Price History[]' => scraper { process '/tr/td[1]', 'key', 'TEXT'; process '/tr/td[2]', 'value', 'TEXT'; }, process q~//table[@id='yfncsumtab']/tr[2]/td/table[2]/tr/td/table/ +tr~, 'Valuation Measures[]' => scraper { process '/tr/td[1]', 'key', 'TEXT'; process '/tr/td[2]', 'value', 'TEXT'; }, }; my $res = $scraper->scrape( @_ ); print Data::Dumper->new([ $res ])->Sortkeys(1)->Dump; } __END__ $VAR1 = { ... 'Cash Flow Statement' => [ { 'key' => 'Cash Flow Statement' }, { 'key' => 'Operating Cash Flow (ttm):', 'value' => '6.30M' }, { 'key' => 'Levered Free Cash Flow (ttm):', 'value' => '3.66M' } ], ...
Come to think of it, Web::Scraper might be also be a bit of a PITA, but I've only studied the trivial examples, not the others

Log In?

What's my password?
Create A New User
Node Status?
node history
Node Type: note [id://891181]
and all is quiet...

How do I use this? | Other CB clients
Other Users?
Others pondering the Monastery: (3)
As of 2017-04-29 18:21 GMT
Find Nodes?
    Voting Booth?
    I'm a fool:

    Results (534 votes). Check out past polls.