Beefy Boxes and Bandwidth Generously Provided by pair Networks
Problems? Is your data what you think it is?
 
PerlMonks  

Re^2: Help With Online Table Scraper

by Anonymous Monk
on Mar 03, 2011 at 09:40 UTC ( #891181=note: print w/ replies, xml ) Need Help??


in reply to Re: Help With Online Table Scraper
in thread Help With Online Table Scraper

Sinistral is right, use a documented API whenever available, scraping is a fragile PITA :)

#!/usr/bin/perl -- use strict; use warnings; use Data::Dumper; use Web::Scraper; use File::Slurp qw' read_file '; use URI; Main( @ARGV ); exit( 0 ); sub Main { my $uri = URI->new('http://finance.yahoo.com/q/ks?s=MNDO+Key+Stati +stics'); if(@_){ my $html_content = read_file( @_ ); ScrapePrint(\$html_content, $uri ); } else { ScrapePrint( $uri ); } } sub ScrapePrint { my $scraper = scraper { process q~//table[@id='yfncsumtab']/tr/td/table[8]//tr//tr~, 'Balance Sheet[]' => scraper { process '/tr/td[1]', 'key', 'TEXT'; process '/tr/td[2]', 'value', 'TEXT'; }, process #~ q~//table[@id='yfncsumtab']/tr/td/table[9]//tr//tr~, q~//table[@id='yfncsumtab']//table[9]//table//tr~, 'Cash Flow Statement[]' => scraper { process '/tr/td[1]', 'key', 'TEXT'; process '/tr/td[2]', 'value', 'TEXT'; }, process q~//table[@id='yfncsumtab']/tr[2]/td[3]/table[4]//tr//tr~, 'Dividends & Splits[]' => scraper { process '/tr/td[1]', 'key', 'TEXT'; process '/tr/td[2]', 'value', 'TEXT'; }, process q~//table[@id='yfncsumtab']/tr/td/table[4]//tr//tr~, 'Fiscal Year[]' => scraper { process '/tr/td[1]', 'key', 'TEXT'; process '/tr/td[2]', 'value', 'TEXT'; }, process q~//table[@id='yfncsumtab']/tr/td/table[7]//tr//tr~, 'Income Statement[]' => scraper { process '/tr/td[1]', 'key', 'TEXT'; process '//tr/td[2]', 'value.', 'TEXT'; }, process q~//table[@id='yfncsumtab']/tr/td/table[6]//tr//tr~, 'Management Effectiveness[]' => scraper { process '/tr/td[1]', 'key', 'TEXT'; process '/tr/td[2]', 'value', 'TEXT'; }, process q~//table[@id='yfncsumtab']/tr/td/table[5]//tr//tr~, 'Profitability[]' => scraper { process '/tr/td[1]', 'key', 'TEXT'; process '/tr/td[2]', 'value', 'TEXT'; }, process q~//table[@id='yfncsumtab']/tr/td[3]/table[3]//tr//tr~, 'Share Statistics[]' => scraper { process '/tr/td[1]', 'key', 'TEXT'; process '/tr/td[2]', 'value', 'TEXT'; }, process q~//table[@id='yfncsumtab']/tr[2]/td[3]/table[2]/tr//tr~, 'Stock Price History[]' => scraper { process '/tr/td[1]', 'key', 'TEXT'; process '/tr/td[2]', 'value', 'TEXT'; }, process q~//table[@id='yfncsumtab']/tr[2]/td/table[2]/tr/td/table/ +tr~, 'Valuation Measures[]' => scraper { process '/tr/td[1]', 'key', 'TEXT'; process '/tr/td[2]', 'value', 'TEXT'; }, }; my $res = $scraper->scrape( @_ ); print Data::Dumper->new([ $res ])->Sortkeys(1)->Dump; } __END__ $VAR1 = { ... 'Cash Flow Statement' => [ { 'key' => 'Cash Flow Statement' }, { 'key' => 'Operating Cash Flow (ttm):', 'value' => '6.30M' }, { 'key' => 'Levered Free Cash Flow (ttm):', 'value' => '3.66M' } ], ...
Come to think of it, Web::Scraper might be also be a bit of a PITA, but I've only studied the trivial examples, not the others


Comment on Re^2: Help With Online Table Scraper
Download Code

Log In?
Username:
Password:

What's my password?
Create A New User
Node Status?
node history
Node Type: note [id://891181]
help
Chatterbox?
and the web crawler heard nothing...

How do I use this? | Other CB clients
Other Users?
Others about the Monastery: (8)
As of 2015-07-05 04:54 GMT
Sections?
Information?
Find Nodes?
Leftovers?
    Voting Booth?

    The top three priorities of my open tasks are (in descending order of likelihood to be worked on) ...









    Results (60 votes), past polls