Scrape News From Hacker News website
- A script that scrapes a number of pages from HackerNews
Run the Script:
- In command go to the file directory, and “run python main.py” in commandline
python main.py
Code language: CSS (css)
Source Code:
main.py
import requests
import os
from bs4 import BeautifulSoup, SoupStrainer
# Makes Output Directory if it does not exist
if not os.path.exists(os.path.join(os.getcwd(), 'HackerNews')):
os.makedirs(os.path.join(os.getcwd(), 'HackerNews'))
'''
@params page_no: The page number of HackerNews to fetch.
Adding only page number in order to add multiprocess support in future.
@params verbose: Adds verbose output to screen instead
of running the program silently.
'''
def fetch(page_no, verbose=False):
# Should be unreachable, but just in case
if page_no <= 0:
raise ValueError('Number of Pages must be greater than zero')
page_no = min(page_no, 20)
i = page_no
if verbose:
print('Fetching Page {}...'.format(i))
try:
res = requests.get('https://news.ycombinator.com/?p=' + str(i))
only_td = SoupStrainer('td')
soup = BeautifulSoup(res.content, 'html.parser', parse_only=only_td)
tdtitle = soup.find_all('td', attrs={'class': 'title'})
tdmetrics = soup.find_all('td', attrs={'class': 'subtext'})
with open(os.path.join('HackerNews', 'NewsPage{}.txt'.format(i)), 'w+') as f:
f.write('-' * 80)
f.write('\n')
f.write('Page {}'.format(i))
tdtitle = soup.find_all('td', attrs={'class': 'title'})
tdrank = soup.find_all(
'td',
attrs={
'class': 'title',
'align': 'right'})
tdtitleonly = [t for t in tdtitle if t not in tdrank]
tdmetrics = soup.find_all('td', attrs={'class': 'subtext'})
tdt = tdtitleonly
tdr = tdrank
tdm = tdmetrics
num_iter = min(len(tdr), len(tdt))
for idx in range(num_iter):
f.write('\n' + '-' * 80 + '\n')
rank = tdr[idx].find('span', attrs={'class': 'rank'})
titl = tdt[idx].find('a', attrs={'class': 'storylink'})
url = titl['href'] if titl and titl['href'].startswith(
'https') else 'https://news.ycombinator.com/' + titl['href']
site = tdt[idx].find('span', attrs={'class': 'sitestr'})
score = tdm[idx].find('span', attrs={'class': 'score'})
time = tdm[idx].find('span', attrs={'class': 'age'})
author = tdm[idx].find('a', attrs={'class': 'hnuser'})
f.write(
'\nArticle Number: ' +
rank.text.replace(
'.',
'') if rank else '\nArticle Number: Could not get article number')
f.write(
'\nArticle Title: ' +
titl.text if titl else '\nArticle Title: Could not get article title')
f.write(
'\nSource Website: ' +
site.text if site else '\nSource Website: https://news.ycombinator.com')
f.write(
'\nSource URL: ' +
url if url else '\nSource URL: No URL found for this article')
f.write(
'\nArticle Author: ' +
author.text if author else '\nArticle Author: Could not get article author')
f.write(
'\nArticle Score: ' +
score.text if score else '\nArticle Score: Not Scored')
f.write(
'\nPosted: ' +
time.text if time else '\nPosted: Could not find when the article was posted')
f.write('\n' + '-' * 80 + '\n')
except (requests.ConnectionError, requests.packages.urllib3.exceptions.ConnectionError) as e:
print('Connection Failed for page {}'.format(i))
except requests.RequestException as e:
print("Some ambiguous Request Exception occurred. The exception is " + str(e))
while(True):
try:
pages = int(
input('Enter number of pages that you want the HackerNews for (max 20): '))
v = input('Want verbose output y/[n] ?')
verbose = v.lower().startswith('y')
if pages > 20:
print('A maximum of only 20 pages can be fetched')
pages = min(pages, 20)
for page_no in range(1, pages + 1):
fetch(page_no, verbose)
break
except ValueError:
print('\nInvalid input, probably not a positive integer\n')
continue
Code language: PHP (php)