Beefy Boxes and Bandwidth Generously Provided by pair Networks
XP is just a number
 
PerlMonks  

Re^2: Help With Online Table Scraper

by Anonymous Monk
on Mar 03, 2011 at 09:40 UTC ( #891181=note: print w/ replies, xml ) Need Help??


in reply to Re: Help With Online Table Scraper
in thread Help With Online Table Scraper

Sinistral is right, use a documented API whenever available, scraping is a fragile PITA :)

#!/usr/bin/perl -- use strict; use warnings; use Data::Dumper; use Web::Scraper; use File::Slurp qw' read_file '; use URI; Main( @ARGV ); exit( 0 ); sub Main { my $uri = URI->new('http://finance.yahoo.com/q/ks?s=MNDO+Key+Stati +stics'); if(@_){ my $html_content = read_file( @_ ); ScrapePrint(\$html_content, $uri ); } else { ScrapePrint( $uri ); } } sub ScrapePrint { my $scraper = scraper { process q~//table[@id='yfncsumtab']/tr/td/table[8]//tr//tr~, 'Balance Sheet[]' => scraper { process '/tr/td[1]', 'key', 'TEXT'; process '/tr/td[2]', 'value', 'TEXT'; }, process #~ q~//table[@id='yfncsumtab']/tr/td/table[9]//tr//tr~, q~//table[@id='yfncsumtab']//table[9]//table//tr~, 'Cash Flow Statement[]' => scraper { process '/tr/td[1]', 'key', 'TEXT'; process '/tr/td[2]', 'value', 'TEXT'; }, process q~//table[@id='yfncsumtab']/tr[2]/td[3]/table[4]//tr//tr~, 'Dividends & Splits[]' => scraper { process '/tr/td[1]', 'key', 'TEXT'; process '/tr/td[2]', 'value', 'TEXT'; }, process q~//table[@id='yfncsumtab']/tr/td/table[4]//tr//tr~, 'Fiscal Year[]' => scraper { process '/tr/td[1]', 'key', 'TEXT'; process '/tr/td[2]', 'value', 'TEXT'; }, process q~//table[@id='yfncsumtab']/tr/td/table[7]//tr//tr~, 'Income Statement[]' => scraper { process '/tr/td[1]', 'key', 'TEXT'; process '//tr/td[2]', 'value.', 'TEXT'; }, process q~//table[@id='yfncsumtab']/tr/td/table[6]//tr//tr~, 'Management Effectiveness[]' => scraper { process '/tr/td[1]', 'key', 'TEXT'; process '/tr/td[2]', 'value', 'TEXT'; }, process q~//table[@id='yfncsumtab']/tr/td/table[5]//tr//tr~, 'Profitability[]' => scraper { process '/tr/td[1]', 'key', 'TEXT'; process '/tr/td[2]', 'value', 'TEXT'; }, process q~//table[@id='yfncsumtab']/tr/td[3]/table[3]//tr//tr~, 'Share Statistics[]' => scraper { process '/tr/td[1]', 'key', 'TEXT'; process '/tr/td[2]', 'value', 'TEXT'; }, process q~//table[@id='yfncsumtab']/tr[2]/td[3]/table[2]/tr//tr~, 'Stock Price History[]' => scraper { process '/tr/td[1]', 'key', 'TEXT'; process '/tr/td[2]', 'value', 'TEXT'; }, process q~//table[@id='yfncsumtab']/tr[2]/td/table[2]/tr/td/table/ +tr~, 'Valuation Measures[]' => scraper { process '/tr/td[1]', 'key', 'TEXT'; process '/tr/td[2]', 'value', 'TEXT'; }, }; my $res = $scraper->scrape( @_ ); print Data::Dumper->new([ $res ])->Sortkeys(1)->Dump; } __END__ $VAR1 = { ... 'Cash Flow Statement' => [ { 'key' => 'Cash Flow Statement' }, { 'key' => 'Operating Cash Flow (ttm):', 'value' => '6.30M' }, { 'key' => 'Levered Free Cash Flow (ttm):', 'value' => '3.66M' } ], ...
Come to think of it, Web::Scraper might be also be a bit of a PITA, but I've only studied the trivial examples, not the others


Comment on Re^2: Help With Online Table Scraper
Download Code

Log In?
Username:
Password:

What's my password?
Create A New User
Node Status?
node history
Node Type: note [id://891181]
help
Chatterbox?
and the web crawler heard nothing...

How do I use this? | Other CB clients
Other Users?
Others avoiding work at the Monastery: (5)
As of 2014-09-18 03:47 GMT
Sections?
Information?
Find Nodes?
Leftovers?
    Voting Booth?

    How do you remember the number of days in each month?











    Results (105 votes), past polls