Add view count to each thread

.github/PULL_REQUEST_TEMPLATE.md

@@ -1 +1,9 @@
-###### Motivation for this change
+
+**What this PR does / why we need it:**
+-
+
+**Which issue(s) this PR fixes:**
+-
+
+**Additional Notes:**
+-

.github/workflows/codeql-analysis.yml

@@ -1,27 +0,0 @@
-name: "Code scanning - action"
-
-on:
-  push:
-  pull_request:
-  schedule:
-    - cron: '0 4 * * 1'
-
-jobs:
-  CodeQL-Build:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
-      with:
-        fetch-depth: 2
-
-    - run: git checkout HEAD^2
-      if: ${{ github.event_name == 'pull_request' }}
-
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1

.gitignore

@@ -88,5 +88,3 @@ venv.bak/
 .mypy_cache/
 .dmypy.json
 dmypy.json
-
-tmp/

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.5.0
+    rev: v2.3.0
     hooks:
     -   id: check-added-large-files
     -   id: check-ast

.travis.yml

@@ -1,5 +1,5 @@
 language: python
-dist: bionic
+dist: xenial
 sudo: false
 cache: false
 stages:
@@ -18,14 +18,15 @@ script:
 jobs:
   include:
     - python: "2.7"
+
     - python: "3.5"
+
     - python: "3.6"
+
     - python: "3.7"
-    - python: "3.8"
-    - python: "3.9-dev"
 
     - stage: deploy
-      python: "3.8"
+      python: "3.7"
       deploy:
         on:
           repo: davegallant/rfd

Makefile

@@ -1,57 +1,35 @@
 
-SRC := rfd
+SRC:=rfd
-SHELL := bash
-.ONESHELL:
-.SHELLFLAGS := -eu -o pipefail -c
-.DELETE_ON_ERROR:
-MAKEFLAGS += --warn-undefined-variables
-MAKEFLAGS += --no-builtin-rules
 
-
-ifeq ($(origin .RECIPEPREFIX), undefined)
-  $(error This Make does not support .RECIPEPREFIX. Please use GNU Make 4.0 or later)
-endif
-.RECIPEPREFIX = >
-
-## build: Build a tar.gz of the python package
-build:
-> rm -rf dist/
-> python setup.py sdist
 .PHONY: build
+build:
+	rm -rf dist/
+	python setup.py sdist
+
+.PHONY: push_test
+push_test:
+	twine upload -r testpypi dist/*.tar.gz
+
+.PHONY: push_prod
+push_prod:
+	twine upload dist/*.tar.gz
 
-## precommit: Run all pre-commit hooks
-precommit:
-> pre-commit run \
- --all-files \
- --show-diff-on-failure
 .PHONY: precommit
+precommit: ## Run pre-commit
+	pre-commit run \
+	--all-files \
+	--show-diff-on-failure
 
-## lint: Run static analysis on the code
-lint:
-> pylint $(SRC)
 .PHONY: lint
+lint:
+	pylint $(SRC)
 
-## test: Run all unit tests
-test: tmp/.tests-passed.sentinel
 .PHONY: test
+test:
+	pytest -v
 
-# Tests - re-ran if any file under src has been changed since tmp/.tests-passed.sentinel was last touched
-tmp/.tests-passed.sentinel: $(shell find ${SRC} -type f)
-> mkdir -p $(@D)
-> pytest -v
-> touch $@
-
-## pr: Run pre-commit, lint and test
-pr: precommit lint test
 .PHONY: pr
+pr: precommit lint test
 
-ci: lint test
 .PHONY: ci
-
+ci: lint test
-## help: Print this help message
-help:
-> @echo "Usage:"
-> @echo
-> @sed -n 's/^##//p' ${MAKEFILE_LIST} | column -t -s ':' |  sed -e 's/^/ /' | sort
-> @echo
-.PHONY: help

README.md

@@ -5,11 +5,11 @@ Hot deals on the command line.
 [![Build Status](https://travis-ci.org/davegallant/rfd.svg?branch=master)](https://travis-ci.org/davegallant/rfd)
 [![PyPI version](https://badge.fury.io/py/rfd.svg)](https://badge.fury.io/py/rfd)
 [![Dependabot](https://badgen.net/badge/Dependabot/enabled/green?icon=dependabot)](https://dependabot.com/)
+[![Total alerts](https://img.shields.io/lgtm/alerts/g/davegallant/rfd.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/davegallant/rfd/alerts/)
 [![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/davegallant/rfd.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/davegallant/rfd/context:python)
 
 
-![screenshot](https://user-images.githubusercontent.com/4519234/85969861-e10a4100-b996-11ea-9a31-6203322c60ee.png)
+![image](https://user-images.githubusercontent.com/4519234/71054408-e18c6a00-211f-11ea-89bc-3f990a4909de.png)
-
 
 ## Install
 
@@ -17,65 +17,29 @@ Hot deals on the command line.
 pip install rfd
 ```
 
-## Usage
+## Use
 
-```shell
+### view threads
-Usage: rfd [OPTIONS] COMMAND [ARGS]...
+```bash
-
+rfd threads [--forum-id 9] [--limit 10]
-  CLI for https://forums.redflagdeals.com
-
-Options:
-  -v, --version
-  --help         Show this message and exit.
-
-Commands:
-  posts    Display all posts in a thread.
-  search   Search deals based on a regular expression.
-  threads  Displays threads in the forum. Defaults to hot deals.
 ```
 
-## Examples
+### search
-
+```bash
-### View Hot Deals
+rfd search pizza [--num-pages 100]
-```console
-$ rfd threads
 ```
 
-### View and Sort Hot Deals
+## Support Tab Completion
-
-```console
-$ rfd threads --sort-by score
-```
-
-```console
-$ rfd threads --sort-by views --limit 40
-```
-
-### Simple Search
-```console
-$ rfd search 'pizza'
-```
-
-### RegEx Search
-
-Regular expressions can be used for search.
-
-```console
-$ rfd search '(coffee|starbucks)' --num-pages 100
-```
-
-## Shell Completion
-
-Completion can be enabled if using `bash` or `zsh`.
 
 ### bash
 
-```console
+```bash
-$ echo 'eval "$(_RFD_COMPLETE=source rfd)"' >> ~/.profile
+echo 'eval "$(_RFD_COMPLETE=source rfd)"' >> ~/.profile
 ```
 
 ### zsh
 
-```console
+
-$ echo 'eval "$(_RFD_COMPLETE=source_zsh rfd)"' >> ~/.zshrc
+```zsh
+echo 'eval "$(_RFD_COMPLETE=source_zsh rfd)"' >> ~/.zshrc
 ```

requirements.txt

@@ -1,5 +1,4 @@
-beautifulsoup4<=4.8.2
+beautifulsoup4>=4.8.1
 click>=7.0
 colorama>=0.4.3
 requests>=2.22.0
-soupsieve<=2.0

requirements_dev.txt

@@ -1,5 +1,5 @@
-pre-commit==1.21.0
+pre-commit==1.20.0
 pylint
 pytest>=4.6.6
-rope==0.16.0
+rope==0.14.0
 tox-travis==0.12

rfd/VERSION

@@ -1 +1 @@
-0.4.1
+0.3.4

rfd/__init__.py

@@ -6,5 +6,3 @@ __title__ = "RFD CLI"
 __author__ = "Dave Gallant"
 __license__ = "Apache 2.0"
 __copyright__ = "(c) 2018 Dave Gallant"
-
-API_BASE_URL = "https://forums.redflagdeals.com"

rfd/api.py

@@ -6,10 +6,11 @@ except ImportError:
     JSONDecodeError = ValueError
 import logging
 import requests
-from . import API_BASE_URL
+from .constants import API_BASE_URL
-from .posts import Post
+from .format import strip_html, is_valid_url
+from .models import Post
 from .scores import calculate_score
-from .utils import is_int, strip_html, is_valid_url
+from .utils import is_int
 
 
 def extract_post_id(url):

rfd/cli.py

@@ -7,7 +7,8 @@ import sys
 import click
 from colorama import init, Fore, Style
 from .api import get_threads, get_posts
-from .threads import parse_threads, search_threads, sort_threads
+from .search import search_threads
+from .parsing import parse_threads
 from .__version__ import version as current_version
 
 init()
@@ -19,7 +20,7 @@ logging.getLogger().addHandler(logging.StreamHandler())
 
 
 def get_version():
-    return "rfd v" + current_version
+    return "rfd " + current_version
 
 
 def get_terminal_width():
@@ -35,61 +36,34 @@ def get_vote_color(score):
     return Fore.BLUE + " [" + str(score) + "] "
 
 
-def print_version(ctx, value):
-    if not value or ctx.resilient_parsing:
-        return
-    click.echo(get_version(), nl=False)
-    ctx.exit()
-
-
-def display_thread(click, thread, count):  # pylint: disable=redefined-outer-name
-    dealer = thread.dealer_name
-    if dealer and dealer is not None:
-        dealer = "[" + dealer + "] "
-    else:
-        dealer = ""
-    click.echo(
-        " "
-        + str(count)
-        + "."
-        + get_vote_color(thread.score)
-        + Fore.RESET
-        + "%s%s" % (dealer, thread.title)
-        + Fore.LIGHTYELLOW_EX
-        + " (%d views)" % thread.views
-        + Fore.RESET
-    )
-    click.echo(Fore.BLUE + " {}".format(thread.url))
-    click.echo(Style.RESET_ALL)
-
-
 @click.group(invoke_without_command=True)
-@click.option(
+@click.option("--version/--no-version", default=False)
-    "-v",
-    "--version",
-    is_flag=True,
-    callback=print_version,
-    expose_value=False,
-    is_eager=True,
-)
 @click.pass_context
-def cli(ctx):
+def cli(ctx, version):
     """CLI for https://forums.redflagdeals.com"""
-    if not ctx.invoked_subcommand:
+    if version:
+        click.echo(get_version())
+    elif not ctx.invoked_subcommand:
         click.echo(ctx.get_help())
 
 
-@cli.command(short_help="Display all posts in a thread.")
+@cli.command("version")
+def display_version():
+    click.echo(get_version())
+
+
+@cli.command(short_help="Displays posts in a specific thread.")
 @click.argument("post_id")
 def posts(post_id):
-    """Iterate all pages and display all posts in a thread.
+    """Displays posts in a specific thread.
 
     post_id can be a full url or post id only
 
     Example:
 
     \b
-    rfd posts https://forums.redflagdeals.com/koodo-targeted-public-mobile-12-120-koodo-5gb-40-no-referrals-2173603
+    url: https://forums.redflagdeals.com/koodo-targeted-public-mobile-12-120-koodo-5gb-40-no-referrals-2173603
+    post_id: 2173603
     """
 
     try:
@@ -113,12 +87,11 @@ def posts(post_id):
         sys.exit(1)
 
 
-@cli.command(short_help="Displays threads in the forum. Defaults to hot deals.")
+@cli.command(short_help="Displays threads in the specified forum.")
+@click.option("--limit", default=10, help="Number of topics.")
 @click.option("--forum-id", default=9, help="The forum id number")
-@click.option("--limit", default=10, help="Number of threads.")
+def threads(limit, forum_id):
-@click.option("--sort-by", default=None, help="Sort threads by")
+    """Displays threads in the specified forum id. Defaults to 9.
-def threads(limit, forum_id, sort_by):
-    """Display threads in the specified forum id. Defaults to 9 (hot deals).
 
     Popular forum ids:
 
@@ -134,21 +107,31 @@ def threads(limit, forum_id, sort_by):
     74 \t shopping discussion
     88 \t cell phones
     """
-    _threads = sort_threads(
+    _threads = parse_threads(get_threads(forum_id, limit), limit)
-        parse_threads(get_threads(forum_id, limit), limit), sort_by=sort_by
-    )
     for count, thread in enumerate(_threads, 1):
-        display_thread(click, thread, count)
+        click.echo(
+            " "
+            + str(count)
+            + "."
+            + get_vote_color(thread.score)
+            + Fore.RESET
+            + "[%s] %s" % (thread.dealer_name, thread.title)
+            + Fore.LIGHTYELLOW_EX
+            + " (%d views)" % thread.total_views
+            + Fore.RESET
+        )
+        click.echo(Fore.BLUE + " {}".format(thread.url))
+        click.echo(Style.RESET_ALL)
 
 
-@cli.command(short_help="Search deals based on a regular expression.")
+@cli.command(short_help="Displays threads in the specified forum.")
 @click.option("--num-pages", default=5, help="Number of pages to search.")
 @click.option(
     "--forum-id", default=9, help="The forum id number. Defaults to 9 (hot deals)."
 )
-@click.argument("regex")
+@click.argument("keyword")
-def search(num_pages, forum_id, regex):
+def search(num_pages, forum_id, keyword):
-    """Search deals based on regex.
+    """Searches for deals based on a keyword in the specified forum id.
 
     Popular forum ids:
 
@@ -168,6 +151,15 @@ def search(num_pages, forum_id, regex):
     count = 0
     for page in range(1, num_pages):
         _threads = parse_threads(get_threads(forum_id, 100, page=page), limit=100)
-        for thread in search_threads(threads=_threads, regex=regex):
+        for thread in search_threads(threads=_threads, keyword=keyword):
             count += 1
-            display_thread(click, thread, count)
+            click.echo(
+                " "
+                + str(count)
+                + "."
+                + get_vote_color(thread.score)
+                + Fore.RESET
+                + "[%s] %s" % (thread.dealer_name, thread.title)
+            )
+            click.echo(Fore.BLUE + " {}".format(thread.url))
+            click.echo(Style.RESET_ALL)

rfd/constants.py

@@ -0,0 +1 @@
+API_BASE_URL = "https://forums.redflagdeals.com"

rfd/format.py

@@ -0,0 +1,16 @@
+"""Formatting utils"""
+
+try:
+    from urllib.parse import urlparse  # python 3
+except ImportError:
+    from urlparse import urlparse  # python 2
+from bs4 import BeautifulSoup
+
+
+def strip_html(text):
+    return BeautifulSoup(text, "html.parser").get_text()
+
+
+def is_valid_url(url):
+    result = urlparse(url)
+    return all([result.scheme, result.netloc, result.path])

rfd/models.py

@@ -0,0 +1,18 @@
+# pylint: disable=old-style-class
+class Thread:
+    def __init__(self, title, dealer_name, score, url, total_views):
+        self.dealer_name = dealer_name
+        self.score = score
+        self.title = title
+        self.url = url
+        self.total_views = total_views
+
+    def __repr__(self):
+        return "Thread(%s)" % self.title
+
+
+class Post:
+    def __init__(self, body, score, user):
+        self.body = body
+        self.score = score
+        self.user = user

rfd/parsing.py

@@ -0,0 +1,35 @@
+from .constants import API_BASE_URL
+from .scores import calculate_score
+from .models import Thread
+
+
+def build_web_path(slug):
+    return "{}{}".format(API_BASE_URL, slug)
+
+
+def parse_threads(threads, limit):
+    """parse topics list api response into digestible list.
+
+    Arguments:
+        threads {dict} -- topics response from rfd api
+        limit {int} -- limit number of threads returned
+
+    Returns:
+        list(dict) -- digestible list of threads
+    """
+    parsed_threads = []
+    if threads is None:
+        return []
+    for count, topic in enumerate(threads.get("topics"), start=1):
+        if count > limit:
+            break
+        parsed_threads.append(
+            Thread(
+                title=topic.get("title"),
+                dealer_name=topic["offer"].get("dealer_name"),
+                score=calculate_score(topic),
+                url=build_web_path(topic.get("web_path")),
+                total_views=topic.get("total_views"),
+            )
+        )
+    return parsed_threads

rfd/posts.py

@@ -1,8 +0,0 @@
-# pylint: disable=old-style-class
-
-
-class Post:
-    def __init__(self, body, score, user):
-        self.body = body
-        self.score = score
-        self.user = user

rfd/search.py

@@ -0,0 +1,13 @@
+def search_threads(threads, keyword=None):
+    """Match deal title and dealer names with keyword specified."""
+
+    if keyword is None:
+        return
+
+    keyword = str(keyword)
+
+    for deal in threads:
+        if keyword.lower() in deal.title.lower() or (
+            deal.dealer_name and keyword.lower() in deal.dealer_name.lower()
+        ):
+            yield deal

rfd/threads.py

@@ -1,76 +0,0 @@
-import re
-from . import API_BASE_URL
-from .scores import calculate_score
-
-# pylint: disable=old-style-class
-class Thread:
-    def __init__(self, title, dealer_name, score, url, views):
-        self.dealer_name = dealer_name
-        self.score = score
-        self.title = title
-        self.url = url
-        self.views = views
-
-    def __repr__(self):
-        return "Thread(%s)" % self.title
-
-
-def build_web_path(slug):
-    return "{}{}".format(API_BASE_URL, slug)
-
-
-def get_dealer(topic):
-    dealer = None
-    if topic.get("offer"):
-        dealer = topic.get("offer").get("dealer_name")
-    return dealer
-
-
-def parse_threads(threads, limit):
-    """Parse topics list api response into digestible list.
-
-    Arguments:
-        threads {dict} -- topics response from rfd api
-        limit {int} -- limit number of threads returned
-
-    Returns:
-        list(dict) -- digestible list of threads
-    """
-    parsed_threads = []
-    if threads is None:
-        return []
-    for count, topic in enumerate(threads.get("topics"), start=1):
-        if count > limit:
-            break
-        parsed_threads.append(
-            Thread(
-                title=topic.get("title"),
-                dealer_name=get_dealer(topic),
-                score=calculate_score(topic),
-                url=build_web_path(topic.get("web_path")),
-                views=topic.get("total_views"),
-            )
-        )
-    return parsed_threads
-
-
-def sort_threads(threads, sort_by):
-    """Sort threads by an attribute"""
-    if sort_by is None:
-        return threads
-    assert sort_by in ["views", "score", "title"]
-    threads = sorted(threads, key=lambda x: getattr(x, sort_by), reverse=True)
-    return threads
-
-
-def search_threads(threads, regex):
-    """Match deal title and dealer names with regex specified."""
-
-    regexp = re.compile(str(regex).lower())
-
-    for deal in threads:
-
-        if regexp.search(deal.title.lower()) or (
-            deal.dealer_name and regexp.search(deal.dealer_name.lower())
-        ):
-            yield deal

rfd/utils.py

@@ -1,18 +1,4 @@
 """This module provides utility functions that are used within rfd"""
-try:
-    from urllib.parse import urlparse  # python 2
-except ImportError:
-    from urlparse import urlparse  # python 1
-from bs4 import BeautifulSoup
-
-
-def strip_html(text):
-    return BeautifulSoup(text, "html.parser").get_text()
-
-
-def is_valid_url(url):
-    result = urlparse(url)
-    return all([result.scheme, result.netloc, result.path])
 
 
 def is_int(number):

tests/test_api.py

@@ -1,5 +1,5 @@
 from rfd.api import extract_post_id
-from rfd.threads import build_web_path, parse_threads
+from rfd.parsing import build_web_path, parse_threads
 
 
 def test_build_web_path():

tox.ini

@@ -9,8 +9,3 @@ envlist = py{27,
 passenv = SSH_AUTH_SOCK
 commands =
     make ci
-    rfd --version
-    rfd threads
-    rfd threads --sort-by score
-    rfd search 'pizza'
-    rfd search '(coffee|starbucks)'