More pytests (#7)

* add constants.py

* more tests and robustness.

* try/catch JSONDecodedError.

rfd/__version__.py

@@ -1,4 +1,4 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-__version__ = '0.1.1'
+__version__ = '0.1.2'

rfd/api.py

@@ -1,8 +1,14 @@
 """RFD API."""
 
+try:
+    from json.decoder import JSONDecodeError
+except ImportError:
+    JSONDecodeError = ValueError
+import logging
 from math import ceil
 import requests
 from bs4 import BeautifulSoup
+from rfd.constants import API_BASE_URL
 
 try:
     from urllib.parse import urlparse  # python 3
@@ -11,7 +17,7 @@ except ImportError:
 
 
 def build_web_path(slug):
-    return "https://forums.redflagdeals.com{}".format(slug)
+    return "{}{}".format(API_BASE_URL, slug)
 
 
 def extract_post_id(url):
@@ -70,10 +76,42 @@ def is_valid_url(url):
 
 
 def get_threads(forum_id, limit):
+    """Get threads from rfd api
+
+    Arguments:
+        forum_id {int} -- forum id
+        limit {[type]} -- limit number of threads returned
+
+    Returns:
+        dict -- api response
+    """
+    try:
+        response = requests.get(
+            "{}/api/topics?forum_id={}&per_page={}".format(API_BASE_URL,
+                                                           forum_id,
+                                                           get_safe_per_page(limit)))
+        if response.status_code == 200:
+            return response.json()
+        logging.error("Unable to retrieve threads. %s", response.text)
+    except JSONDecodeError as err:
+        logging.error("Unable to retrieve threads. %s", err)
+    return None
+
+
+def parse_threads(api_response, limit):
+    """parse topics list api response into digestible list.
+
+    Arguments:
+        api_response {dict} -- topics response from rfd api
+        limit {int} -- limit number of threads returned
+
+    Returns:
+        list(dict) -- digestible list of threads
+    """
     threads = []
-    response = requests.get(
+    if api_response is None:
-        "https://forums.redflagdeals.com/api/topics?forum_id={}&per_page={}".format(forum_id, get_safe_per_page(limit)))
+        return threads
-    for topic in response.json().get('topics'):
+    for topic in api_response.get('topics'):
         threads.append({
             'title': topic.get('title'),
             'score': calculate_score(topic),
@@ -100,7 +138,8 @@ def get_posts(post, count=5, tail=False, per_page=40):
         raise ValueError()
 
     response = requests.get(
-        "https://forums.redflagdeals.com/api/topics/{}/posts?per_page=40&page=1".format(post_id))
+        "{}/api/topics/{}/posts?per_page=40&page=1".format(API_BASE_URL,
+                                                           post_id))
     total_posts = response.json().get('pager').get('total')
     total_pages = response.json().get('pager').get('total_pages')
 
@@ -127,10 +166,11 @@ def get_posts(post, count=5, tail=False, per_page=40):
     # Go through as many pages as necessary
     for page in range(start_page, pages + 1):
         response = requests.get(
-            "https://forums.redflagdeals.com/api/topics/{}/posts?per_page={}&page={}".format(post_id,
+            "{}/api/topics/{}/posts?per_page={}&page={}".format(API_BASE_URL,
-                                                                                             get_safe_per_page(
+                                                                post_id,
-                                                                                                 per_page),
+                                                                get_safe_per_page(
-                                                                                             page))
+                                                                    per_page),
+                                                                page))
 
         users = users_to_dict(response.json().get('users'))
 

rfd/constants.py

@@ -0,0 +1,2 @@
+
+API_BASE_URL = 'https://forums.redflagdeals.com'

rfd/rfd_cli.py

@@ -1,16 +1,21 @@
 from __future__ import unicode_literals
 
 
-import sys
+import logging
 import os
+import sys
 import click
 from colorama import init, Fore, Style
-from rfd.api import get_threads, get_posts
+from rfd.api import parse_threads, get_threads, get_posts
 from rfd.__version__ import __version__
 
 init()
 print()
 
+logging.getLogger()
+logging.getLogger().setLevel(logging.INFO)
+logging.getLogger().addHandler(logging.StreamHandler())
+
 
 def get_version():
     return 'rfd ' + __version__
@@ -108,7 +113,7 @@ def threads(count, forum_id):
     74 \t shopping discussion
     88 \t cell phones
     """
-    _threads = get_threads(forum_id, count)
+    _threads = parse_threads(get_threads(forum_id, count), count)
     for i, thread in enumerate(_threads, 1):
         click.echo(" " + str(i) + "." +
                    get_vote_color(thread.get('score')) + Fore.RESET + thread.get('title'))

tests/conftest.py

@@ -0,0 +1,8 @@
+import json
+import pytest
+
+
+@pytest.fixture
+def threads_api_response():
+    with open("tests/mock_data/threads_api_response.json") as json_file:
+        return json.load(json_file)

tests/mock_data/threads_api_response.json

@@ -0,0 +1,782 @@
+{
+  "pager": {
+    "total": -1,
+    "per_page": 10,
+    "total_on_page": 10,
+    "total_pages": -1,
+    "page": 1
+  },
+  "topics": [
+    {
+      "topic_id": 2197859,
+      "forum_id": 9,
+      "title":
+        "[Sponsored] 3 Months Free, Cable 75M, Unlimited Internet $34.99/30 Days, Free Installation/Modem Rental",
+      "slug":
+        "carrytel-sponsored-3-months-free-cable-75m-unlimited-internet-34-99-30-days-free-installation-modem-rental",
+      "author_id": 4437,
+      "is_subscribed": false,
+      "can_subscribe": false,
+      "can_report": false,
+      "can_reply": false,
+      "is_poll": false,
+      "status": 1,
+      "type": 1,
+      "post_time": "2018-06-01T14:46:14+00:00",
+      "last_post_time": "2018-06-01T14:46:14+00:00",
+      "first_post_id": 29395579,
+      "last_post_id": 29395579,
+      "last_poster_id": 4437,
+      "total_replies": 0,
+      "has_new_post": false,
+      "total_views": 2345,
+      "visibility": 1,
+      "has_replied": false,
+      "shadow_type": 0,
+      "web_path":
+        "/carrytel-sponsored-3-months-free-cable-75m-unlimited-internet-34-99-30-days-free-installation-modem-rental-2197859/",
+      "offer": {
+        "category_id": 5125,
+        "dealer_id": 0,
+        "dealer_name": "Carrytel",
+        "url": "https://www.carrytel.ca/3mfree.aspx",
+        "price": "",
+        "savings": "",
+        "expires_at": "2018-06-14"
+      },
+      "votes": null,
+      "prefix": null,
+      "summary": {
+        "can_edit": false,
+        "body": ""
+      },
+      "trader_type": null
+    },
+    {
+      "topic_id": 2191108,
+      "forum_id": 9,
+      "title": "WyzeCam 1080p HD Wireless Smart Home Camera v2 $37.49",
+      "slug": "amazon-ca-wyzecam-1080p-hd-wireless-smart-home-camera-v2-37-49",
+      "author_id": 1411026,
+      "is_subscribed": false,
+      "can_subscribe": false,
+      "can_report": false,
+      "can_reply": false,
+      "is_poll": false,
+      "status": 0,
+      "type": 0,
+      "post_time": "2018-05-03T15:06:55+00:00",
+      "last_post_time": "2018-06-03T02:43:17+00:00",
+      "first_post_id": 29284857,
+      "last_post_id": 29401295,
+      "last_poster_id": 38695,
+      "total_replies": 709,
+      "has_new_post": false,
+      "total_views": 129121,
+      "visibility": 1,
+      "has_replied": false,
+      "shadow_type": 0,
+      "web_path":
+        "/amazon-ca-wyzecam-1080p-hd-wireless-smart-home-camera-v2-37-49-2191108/",
+      "offer": {
+        "category_id": 5123,
+        "dealer_id": 775,
+        "dealer_name": "Amazon Canada",
+        "url": "https://www.amazon.ca/gp/aw/d/B076H3SRXG/",
+        "price": "$37.49",
+        "savings": "",
+        "expires_at": null
+      },
+      "votes": {
+        "total_up": 94,
+        "total_down": 2,
+        "current_vote": 0,
+        "can_vote_up": false,
+        "can_vote_down": false
+      },
+      "prefix": null,
+      "summary": {
+        "can_edit": false,
+        "body": ""
+      },
+      "trader_type": null
+    },
+    {
+      "topic_id": 2197916,
+      "forum_id": 9,
+      "title": "Jabra Elite 65T $169.99",
+      "slug": "best-buy-jabra-elite-65t-169-99",
+      "author_id": 68535,
+      "is_subscribed": false,
+      "can_subscribe": false,
+      "can_report": false,
+      "can_reply": false,
+      "is_poll": false,
+      "status": 0,
+      "type": 0,
+      "post_time": "2018-06-01T18:36:28+00:00",
+      "last_post_time": "2018-06-03T02:41:57+00:00",
+      "first_post_id": 29396697,
+      "last_post_id": 29401292,
+      "last_poster_id": 37990,
+      "total_replies": 37,
+      "has_new_post": false,
+      "total_views": 6171,
+      "visibility": 1,
+      "has_replied": false,
+      "shadow_type": 0,
+      "web_path": "/best-buy-jabra-elite-65t-169-99-2197916/",
+      "offer": {
+        "category_id": 9,
+        "dealer_id": 57,
+        "dealer_name": "Best Buy",
+        "url":
+          "https://m.bestbuy.ca/en-CA/product/jabra-elite-65t-in-ear-noise-cancelling-truly-wireless-bluetooth-headphones-with-mic-black/12321947",
+        "price": "169.99",
+        "savings": "",
+        "expires_at": "2018-06-07"
+      },
+      "votes": {
+        "total_up": 8,
+        "total_down": 0,
+        "current_vote": 0,
+        "can_vote_up": false,
+        "can_vote_down": false
+      },
+      "prefix": null,
+      "summary": {
+        "can_edit": false,
+        "body":
+          "Desktop / non-mobile link: \u003Ca href=\u0022https://www.bestbuy.ca/en-ca/product/jabra-elite-65t-in-ear-noise-cancelling-truly-wireless-bluetooth-headphones-with-mic-black/12321947.aspx\u0022 rel=\u0022nofollow noreferrer\u0022 target=\u0022_blank\u0022 class=\u0022postlink\u0022\u003Ehttps://www.bestbuy.ca/en-ca/product/ja ... 21947.aspx\u003C/a\u003E"
+      },
+      "trader_type": null
+    },
+    {
+      "topic_id": 2198211,
+      "forum_id": 9,
+      "title":
+        "Glad Cling Wrap Plastic Wrap, 300 Metre Roll - best price $9.47",
+      "slug":
+        "amazon-ca-glad-cling-wrap-plastic-wrap-300-metre-roll-best-price-9-47",
+      "author_id": 1408083,
+      "is_subscribed": false,
+      "can_subscribe": false,
+      "can_report": false,
+      "can_reply": false,
+      "is_poll": false,
+      "status": 0,
+      "type": 0,
+      "post_time": "2018-06-03T02:13:45+00:00",
+      "last_post_time": "2018-06-03T02:34:19+00:00",
+      "first_post_id": 29401248,
+      "last_post_id": 29401285,
+      "last_poster_id": 26936,
+      "total_replies": 1,
+      "has_new_post": false,
+      "total_views": 264,
+      "visibility": 1,
+      "has_replied": false,
+      "shadow_type": 0,
+      "web_path":
+        "/amazon-ca-glad-cling-wrap-plastic-wrap-300-metre-roll-best-price-9-47-2198211/",
+      "offer": {
+        "category_id": 5142,
+        "dealer_id": 775,
+        "dealer_name": "Amazon Canada",
+        "url":
+          "https://www.amazon.ca/Glad-Cling-Wrap-Plastic-Metre/dp/B01GZTGTWW/ref=sr_1_6?ie=UTF8\u0026amp;qid=1527978057\u0026amp;sr=8-6\u0026amp;keywords=cling+film",
+        "price": "9.47",
+        "savings": "28% off",
+        "expires_at": null
+      },
+      "votes": {
+        "total_up": 1,
+        "total_down": 0,
+        "current_vote": 0,
+        "can_vote_up": false,
+        "can_vote_down": false
+      },
+      "prefix": null,
+      "summary": {
+        "can_edit": false,
+        "body": ""
+      },
+      "trader_type": null
+    },
+    {
+      "topic_id": 2195171,
+      "forum_id": 9,
+      "title": "Firman 3300 inverter generator $599",
+      "slug": "costco-firman-3300-inverter-generator-599",
+      "author_id": 1364941,
+      "is_subscribed": false,
+      "can_subscribe": false,
+      "can_report": false,
+      "can_reply": false,
+      "is_poll": false,
+      "status": 0,
+      "type": 0,
+      "post_time": "2018-05-21T07:29:13+00:00",
+      "last_post_time": "2018-06-03T02:33:41+00:00",
+      "first_post_id": 29351891,
+      "last_post_id": 29401283,
+      "last_poster_id": 69083,
+      "total_replies": 124,
+      "has_new_post": false,
+      "total_views": 18634,
+      "visibility": 1,
+      "has_replied": false,
+      "shadow_type": 0,
+      "web_path": "/costco-firman-3300-inverter-generator-599-2195171/",
+      "offer": {
+        "category_id": 5136,
+        "dealer_id": 110,
+        "dealer_name": "Costco",
+        "url": "https://m.costco.ca/.product.2414104.html",
+        "price": "$599",
+        "savings": "",
+        "expires_at": "2018-06-03"
+      },
+      "votes": {
+        "total_up": 17,
+        "total_down": 0,
+        "current_vote": 0,
+        "can_vote_up": false,
+        "can_vote_down": false
+      },
+      "prefix": null,
+      "summary": {
+        "can_edit": false,
+        "body": ""
+      },
+      "trader_type": null
+    },
+    {
+      "topic_id": 2198199,
+      "forum_id": 9,
+      "title": "HOT - KitchenAid Stand Mixer - $199",
+      "slug": "walmart-hot-kitchenaid-stand-mixer-199",
+      "author_id": 265861,
+      "is_subscribed": false,
+      "can_subscribe": false,
+      "can_report": false,
+      "can_reply": false,
+      "is_poll": false,
+      "status": 0,
+      "type": 0,
+      "post_time": "2018-06-03T00:05:40+00:00",
+      "last_post_time": "2018-06-03T02:33:04+00:00",
+      "first_post_id": 29400944,
+      "last_post_id": 29401282,
+      "last_poster_id": 311667,
+      "total_replies": 8,
+      "has_new_post": false,
+      "total_views": 1334,
+      "visibility": 1,
+      "has_replied": false,
+      "shadow_type": 0,
+      "web_path": "/walmart-hot-kitchenaid-stand-mixer-199-2198199/",
+      "offer": {
+        "category_id": 5139,
+        "dealer_id": 450,
+        "dealer_name": "Walmart",
+        "url":
+          "https://www.walmart.ca/en/ip/kitchenaid-275w-classic-plus-stand-mixer-ksm75sl/6000195341286",
+        "price": "199",
+        "savings": "50% off",
+        "expires_at": null
+      },
+      "votes": {
+        "total_up": 4,
+        "total_down": 1,
+        "current_vote": 0,
+        "can_vote_up": false,
+        "can_vote_down": false
+      },
+      "prefix": null,
+      "summary": {
+        "can_edit": false,
+        "body": ""
+      },
+      "trader_type": null
+    },
+    {
+      "topic_id": 2198164,
+      "forum_id": 9,
+      "title":
+        "Seagate Expansion 4TB Portable External Hard Drive USB 3.0 (STEA4000400) $119.92",
+      "slug":
+        "amazon-ca-seagate-expansion-4tb-portable-external-hard-drive-usb-3-0-stea4000400-119-92",
+      "author_id": 177843,
+      "is_subscribed": false,
+      "can_subscribe": false,
+      "can_report": false,
+      "can_reply": false,
+      "is_poll": false,
+      "status": 0,
+      "type": 0,
+      "post_time": "2018-06-02T20:12:30+00:00",
+      "last_post_time": "2018-06-03T02:32:09+00:00",
+      "first_post_id": 29400374,
+      "last_post_id": 29401280,
+      "last_poster_id": 1373719,
+      "total_replies": 4,
+      "has_new_post": false,
+      "total_views": 827,
+      "visibility": 1,
+      "has_replied": false,
+      "shadow_type": 0,
+      "web_path":
+        "/amazon-ca-seagate-expansion-4tb-portable-external-hard-drive-usb-3-0-stea4000400-119-92-2198164/",
+      "offer": {
+        "category_id": 5119,
+        "dealer_id": 775,
+        "dealer_name": "Amazon Canada",
+        "url":
+          "https://www.amazon.ca/Seagate-Expansion-Portable-External-STEA4000400/dp/B017KE8OG0/ref=pd_cart_crc_sbs_2_1?_encoding=UTF8\u0026amp;pd_rd_i=B017KE8OG0\u0026amp;pd_rd_r=19RNAC5G5MN50M7THMXQ\u0026amp;pd_rd_w=NPMyB\u0026amp;pd_rd_wg=2y64K\u0026amp;pf_rd_i=cart-page-widgets\u0026amp;pf_rd_m=A3DWYIK6Y9EEQB\u0026amp;pf_rd_p=1228945996396599610\u0026amp;pf_rd_r=19RNAC5G5MN50M7THMXQ\u0026amp;pf_rd_s=cart-page-widgets\u0026amp;pf_rd_t=40701\u0026amp;psc=1\u0026amp;refRID=19RNAC5G5MN50M7THMXQ",
+        "price": "119.92",
+        "savings": "13%",
+        "expires_at": null
+      },
+      "votes": {
+        "total_up": 0,
+        "total_down": 1,
+        "current_vote": 0,
+        "can_vote_up": false,
+        "can_vote_down": false
+      },
+      "prefix": null,
+      "summary": {
+        "can_edit": false,
+        "body": ""
+      },
+      "trader_type": null
+    },
+    {
+      "topic_id": 2198212,
+      "forum_id": 9,
+      "title":
+        "WORKSHOP Wet Dry Vacs Ash Vacuum Cleaner WS0500ASH, 5-Gallon Ash Vac 65% Off, Now: $48.54",
+      "slug":
+        "workshop-wet-dry-vacs-ash-vacuum-cleaner-ws0500ash-5-gallon-ash-vac-65-off-now-48-54",
+      "author_id": 1431114,
+      "is_subscribed": false,
+      "can_subscribe": false,
+      "can_report": false,
+      "can_reply": false,
+      "is_poll": false,
+      "status": 0,
+      "type": 0,
+      "post_time": "2018-06-03T02:25:50+00:00",
+      "last_post_time": "2018-06-03T02:25:50+00:00",
+      "first_post_id": 29401272,
+      "last_post_id": 29401272,
+      "last_poster_id": 1431114,
+      "total_replies": 0,
+      "has_new_post": false,
+      "total_views": 129,
+      "visibility": 1,
+      "has_replied": false,
+      "shadow_type": 0,
+      "web_path":
+        "/workshop-wet-dry-vacs-ash-vacuum-cleaner-ws0500ash-5-gallon-ash-vac-65-off-now-48-54-2198212/",
+      "offer": {
+        "category_id": 5,
+        "dealer_id": 0,
+        "dealer_name": "",
+        "url":
+          "https://www.amazon.ca/WORKSHOP-Wet-Dry-Vacs-WS0500ASH/dp/B01983NYD4/ref=as_li_ss_tl?ie=UTF8\u0026amp;linkCode=ll1\u0026amp;tag=ybbs-20\u0026amp;linkId=d729e8e2fc16a114465332d984ab914d",
+        "price": "",
+        "savings": "",
+        "expires_at": null
+      },
+      "votes": {
+        "total_up": 0,
+        "total_down": 0,
+        "current_vote": 0,
+        "can_vote_up": false,
+        "can_vote_down": false
+      },
+      "prefix": null,
+      "summary": {
+        "can_edit": false,
+        "body": ""
+      },
+      "trader_type": null
+    },
+    {
+      "topic_id": 2198191,
+      "forum_id": 9,
+      "title": "NBA 2K18 (Nintendo Switch) -$19.99 or $16.99 PM",
+      "slug": "the-source-nba-2k18-nintendo-switch-19-99-16-99-pm",
+      "author_id": 446796,
+      "is_subscribed": false,
+      "can_subscribe": false,
+      "can_report": false,
+      "can_reply": false,
+      "is_poll": false,
+      "status": 0,
+      "type": 0,
+      "post_time": "2018-06-02T23:03:52+00:00",
+      "last_post_time": "2018-06-03T02:24:50+00:00",
+      "first_post_id": 29400815,
+      "last_post_id": 29401270,
+      "last_poster_id": 1379113,
+      "total_replies": 1,
+      "has_new_post": false,
+      "total_views": 957,
+      "visibility": 1,
+      "has_replied": false,
+      "shadow_type": 0,
+      "web_path":
+        "/the-source-nba-2k18-nintendo-switch-19-99-16-99-pm-2198191/",
+      "offer": {
+        "category_id": 5127,
+        "dealer_id": 491,
+        "dealer_name": "The Source",
+        "url":
+          "https://www.thesource.ca/en-ca/gaming/nintendo-switch/nintendo-switch-games/nba-2k18-for-nintendo-switch/p/108070285",
+        "price": "$19.99",
+        "savings": "",
+        "expires_at": null
+      },
+      "votes": {
+        "total_up": 4,
+        "total_down": 0,
+        "current_vote": 0,
+        "can_vote_up": false,
+        "can_vote_down": false
+      },
+      "prefix": null,
+      "summary": {
+        "can_edit": false,
+        "body": ""
+      },
+      "trader_type": null
+    },
+    {
+      "topic_id": 2198145,
+      "forum_id": 9,
+      "title": "CROCS.CA 40%OFF Select style - at checkout",
+      "slug": "crocs-crocs-ca-40-off-select-style-checkout",
+      "author_id": 690944,
+      "is_subscribed": false,
+      "can_subscribe": false,
+      "can_report": false,
+      "can_reply": false,
+      "is_poll": false,
+      "status": 0,
+      "type": 0,
+      "post_time": "2018-06-02T18:05:22+00:00",
+      "last_post_time": "2018-06-03T02:22:46+00:00",
+      "first_post_id": 29400002,
+      "last_post_id": 29401267,
+      "last_poster_id": 835285,
+      "total_replies": 4,
+      "has_new_post": false,
+      "total_views": 1247,
+      "visibility": 1,
+      "has_replied": false,
+      "shadow_type": 0,
+      "web_path": "/crocs-crocs-ca-40-off-select-style-checkout-2198145/",
+      "offer": {
+        "category_id": 12,
+        "dealer_id": 112,
+        "dealer_name": "Crocs",
+        "url": "https://www.crocs.ca/sale/crocs-deals,en_CA,sc.html",
+        "price": "vary",
+        "savings": "40% off",
+        "expires_at": null
+      },
+      "votes": {
+        "total_up": 5,
+        "total_down": 0,
+        "current_vote": 0,
+        "can_vote_up": false,
+        "can_vote_down": false
+      },
+      "prefix": null,
+      "summary": {
+        "can_edit": false,
+        "body": ""
+      },
+      "trader_type": null
+    }
+  ],
+  "users": [
+    {
+      "user_id": 4437,
+      "username": "RedFlagDeals.com",
+      "total_posts": 7987,
+      "can_view_messages": true,
+      "can_send_messages": true,
+      "is_local_deals_admin": false,
+      "avatar_url": null,
+      "avatar_width": null,
+      "avatar_height": null,
+      "rank": "Moderator",
+      "group": 4,
+      "join_date": "2003-05-18T16:00:00+00:00",
+      "location": ""
+    },
+    {
+      "user_id": 26936,
+      "username": "ahoroba",
+      "total_posts": 1335,
+      "can_view_messages": true,
+      "can_send_messages": true,
+      "is_local_deals_admin": false,
+      "avatar_url":
+        "https://forums.redflagdeals.com/avatar/26936_1470311361.gif",
+      "avatar_width": 32,
+      "avatar_height": 32,
+      "rank": "Deal Addict",
+      "group": 2,
+      "join_date": "2005-07-25T14:30:43+00:00",
+      "location": ""
+    },
+    {
+      "user_id": 37990,
+      "username": "EP32k2",
+      "total_posts": 9184,
+      "can_view_messages": true,
+      "can_send_messages": true,
+      "is_local_deals_admin": false,
+      "avatar_url": null,
+      "avatar_width": null,
+      "avatar_height": null,
+      "rank": "Penalty Box",
+      "group": 2,
+      "join_date": "2006-01-16T22:54:23+00:00",
+      "location": "Richmond Hill"
+    },
+    {
+      "user_id": 38695,
+      "username": "astigdealz",
+      "total_posts": 334,
+      "can_view_messages": true,
+      "can_send_messages": true,
+      "is_local_deals_admin": false,
+      "avatar_url":
+        "https://forums.redflagdeals.com/avatar/38695_1484397258.jpg",
+      "avatar_width": 95,
+      "avatar_height": 100,
+      "rank": "Member",
+      "group": 2,
+      "join_date": "2006-01-29T16:04:31+00:00",
+      "location": "Brampton"
+    },
+    {
+      "user_id": 68535,
+      "username": "vanhalen26",
+      "total_posts": 236,
+      "can_view_messages": true,
+      "can_send_messages": true,
+      "is_local_deals_admin": false,
+      "avatar_url": null,
+      "avatar_width": null,
+      "avatar_height": null,
+      "rank": "Member",
+      "group": 2,
+      "join_date": "2006-12-01T17:32:38+00:00",
+      "location": "Toronto"
+    },
+    {
+      "user_id": 69083,
+      "username": "Imago",
+      "total_posts": 358,
+      "can_view_messages": true,
+      "can_send_messages": true,
+      "is_local_deals_admin": false,
+      "avatar_url": null,
+      "avatar_width": null,
+      "avatar_height": null,
+      "rank": "Member",
+      "group": 2,
+      "join_date": "2006-12-04T19:51:16+00:00",
+      "location": "Craven"
+    },
+    {
+      "user_id": 177843,
+      "username": "voodoo22",
+      "total_posts": 497,
+      "can_view_messages": true,
+      "can_send_messages": true,
+      "is_local_deals_admin": false,
+      "avatar_url": null,
+      "avatar_width": null,
+      "avatar_height": null,
+      "rank": "Member",
+      "group": 2,
+      "join_date": "2008-08-25T16:38:10+00:00",
+      "location": "Victoria"
+    },
+    {
+      "user_id": 265861,
+      "username": "pvdv",
+      "total_posts": 15,
+      "can_view_messages": true,
+      "can_send_messages": true,
+      "is_local_deals_admin": false,
+      "avatar_url": null,
+      "avatar_width": null,
+      "avatar_height": null,
+      "rank": "Newbie",
+      "group": 2,
+      "join_date": "2009-10-28T21:56:08+00:00",
+      "location": "Waterloo, ON, Canada"
+    },
+    {
+      "user_id": 311667,
+      "username": "tew",
+      "total_posts": 1158,
+      "can_view_messages": true,
+      "can_send_messages": true,
+      "is_local_deals_admin": false,
+      "avatar_url":
+        "https://forums.redflagdeals.com/avatar/311667_1470311465.jpg",
+      "avatar_width": 100,
+      "avatar_height": 80,
+      "rank": "Deal Addict",
+      "group": 2,
+      "join_date": "2010-06-14T19:23:15+00:00",
+      "location": "GTA"
+    },
+    {
+      "user_id": 446796,
+      "username": "cain2589",
+      "total_posts": 261,
+      "can_view_messages": true,
+      "can_send_messages": true,
+      "is_local_deals_admin": false,
+      "avatar_url": null,
+      "avatar_width": null,
+      "avatar_height": null,
+      "rank": "Member",
+      "group": 2,
+      "join_date": "2011-07-24T23:56:20+00:00",
+      "location": "Ottawa"
+    },
+    {
+      "user_id": 690944,
+      "username": "Bessem",
+      "total_posts": 833,
+      "can_view_messages": true,
+      "can_send_messages": true,
+      "is_local_deals_admin": false,
+      "avatar_url": null,
+      "avatar_width": null,
+      "avatar_height": null,
+      "rank": "Sr. Member",
+      "group": 2,
+      "join_date": "2013-11-08T16:06:10+00:00",
+      "location": "Montreal"
+    },
+    {
+      "user_id": 835285,
+      "username": "tark514",
+      "total_posts": 250,
+      "can_view_messages": true,
+      "can_send_messages": true,
+      "is_local_deals_admin": false,
+      "avatar_url": null,
+      "avatar_width": null,
+      "avatar_height": null,
+      "rank": "Member",
+      "group": 2,
+      "join_date": "2015-12-06T12:36:38+00:00",
+      "location": "Verdun, QC"
+    },
+    {
+      "user_id": 1364941,
+      "username": "Susanclark",
+      "total_posts": 41,
+      "can_view_messages": true,
+      "can_send_messages": true,
+      "is_local_deals_admin": false,
+      "avatar_url": null,
+      "avatar_width": null,
+      "avatar_height": null,
+      "rank": "Newbie",
+      "group": 2,
+      "join_date": "2016-08-19T07:04:00+00:00",
+      "location": ""
+    },
+    {
+      "user_id": 1373719,
+      "username": "natrill",
+      "total_posts": 52,
+      "can_view_messages": true,
+      "can_send_messages": true,
+      "is_local_deals_admin": false,
+      "avatar_url": null,
+      "avatar_width": null,
+      "avatar_height": null,
+      "rank": "Newbie",
+      "group": 2,
+      "join_date": "2016-11-26T23:55:52+00:00",
+      "location": ""
+    },
+    {
+      "user_id": 1379113,
+      "username": "thyeri",
+      "total_posts": 13,
+      "can_view_messages": true,
+      "can_send_messages": true,
+      "is_local_deals_admin": false,
+      "avatar_url": null,
+      "avatar_width": null,
+      "avatar_height": null,
+      "rank": "Newbie",
+      "group": 2,
+      "join_date": "2017-01-20T04:10:14+00:00",
+      "location": ""
+    },
+    {
+      "user_id": 1408083,
+      "username": "Kyuuan",
+      "total_posts": 81,
+      "can_view_messages": true,
+      "can_send_messages": true,
+      "is_local_deals_admin": false,
+      "avatar_url":
+        "https://forums.redflagdeals.com/avatar/1408083_1511554534.jpg",
+      "avatar_width": 100,
+      "avatar_height": 100,
+      "rank": "Newbie",
+      "group": 2,
+      "join_date": "2017-11-09T02:55:42+00:00",
+      "location": ""
+    },
+    {
+      "user_id": 1411026,
+      "username": "HeyAgain88",
+      "total_posts": 4,
+      "can_view_messages": true,
+      "can_send_messages": true,
+      "is_local_deals_admin": false,
+      "avatar_url": null,
+      "avatar_width": null,
+      "avatar_height": null,
+      "rank": "Newbie",
+      "group": 2,
+      "join_date": "2017-11-27T01:18:10+00:00",
+      "location": ""
+    },
+    {
+      "user_id": 1431114,
+      "username": "askmenow",
+      "total_posts": 6,
+      "can_view_messages": true,
+      "can_send_messages": true,
+      "is_local_deals_admin": false,
+      "avatar_url": null,
+      "avatar_width": null,
+      "avatar_height": null,
+      "rank": "Newbie",
+      "group": 2,
+      "join_date": "2018-05-24T14:34:30+00:00",
+      "location": ""
+    }
+  ]
+}

tests/test_api.py

@@ -1,4 +1,4 @@
-from rfd.api import build_web_path, extract_post_id
+from rfd.api import build_web_path, extract_post_id, parse_threads
 
 
 def test_build_web_path():
@@ -6,5 +6,47 @@ def test_build_web_path():
 
 
 def test_extract_post_id():
-    assert extract_post_id("https://forums.redflagdeals.com/targeted-bob-2173603/120") == '2173603'
+    assert extract_post_id(
-    assert extract_post_id("http://forums.redflagdeals.com/targeted-2173604/120") == '2173604'
+        "https://forums.redflagdeals.com/targeted-bob-2173603/120") == '2173603'
+    assert extract_post_id(
+        "http://forums.redflagdeals.com/targeted-2173604/120") == '2173604'
+
+
+def test_parse_threads(threads_api_response):
+
+    assert len(parse_threads(threads_api_response, 10)) == len(
+        [{'score': 0,
+          'title': '[Sponsored] 3 Months Free, Cable 75M, Unlimited Internet $34.99/30 '
+          'Days, Free Installation/Modem Rental',
+          'url': 'https://forums.redflagdeals.com/carrytel-sponsored-3-months-free-cable-75m-unlimited-internet-34-99-30-days-free-installation-modem-rental-2197859/'},
+         {'score': 92,
+          'title': 'WyzeCam 1080p HD Wireless Smart Home Camera v2 $37.49',
+          'url': 'https://forums.redflagdeals.com/amazon-ca-wyzecam-1080p-hd-wireless-smart-home-camera-v2-37-49-2191108/'},
+         {'score': 8,
+          'title': 'Jabra Elite 65T $169.99',
+          'url': 'https://forums.redflagdeals.com/best-buy-jabra-elite-65t-169-99-2197916/'},
+         {'score': 1,
+          'title': 'Glad Cling Wrap Plastic Wrap, 300 Metre Roll - best price $9.47',
+          'url': 'https://forums.redflagdeals.com/amazon-ca-glad-cling-wrap-plastic-wrap-300-metre-roll-best-price-9-47-2198211/'},
+         {'score': 17,
+          'title': 'Firman 3300 inverter generator $599',
+          'url': 'https://forums.redflagdeals.com/costco-firman-3300-inverter-generator-599-2195171/'},
+         {'score': 3,
+          'title': 'HOT - KitchenAid Stand Mixer - $199',
+          'url': 'https://forums.redflagdeals.com/walmart-hot-kitchenaid-stand-mixer-199-2198199/'},
+         {'score': -1,
+          'title': 'Seagate Expansion 4TB Portable External Hard Drive USB 3.0 '
+          '(STEA4000400) $119.92',
+          'url': 'https://forums.redflagdeals.com/amazon-ca-seagate-expansion-4tb-portable-external-hard-drive-usb-3-0-stea4000400-119-92-2198164/'},
+         {'score': 0,
+          'title': 'WORKSHOP Wet Dry Vacs Ash Vacuum Cleaner WS0500ASH, 5-Gallon Ash '
+          'Vac 65% Off, Now: $48.54',
+          'url': 'https://forums.redflagdeals.com/workshop-wet-dry-vacs-ash-vacuum-cleaner-ws0500ash-5-gallon-ash-vac-65-off-now-48-54-2198212/'},
+         {'score': 4,
+          'title': 'NBA 2K18 (Nintendo Switch) -$19.99 or $16.99 PM',
+          'url': 'https://forums.redflagdeals.com/the-source-nba-2k18-nintendo-switch-19-99-16-99-pm-2198191/'},
+         {'score': 5,
+          'title': 'CROCS.CA 40%OFF Select style - at checkout',
+          'url': 'https://forums.redflagdeals.com/crocs-crocs-ca-40-off-select-style-checkout-2198145/'}])
+
+    assert len(parse_threads(None, 10)) == 0