first commit.

.gitignore

@@ -0,0 +1,9 @@
+**.egg-info/
+**.pyc
+**.pyo
+.pypirc
+.pytest_cache/
+build/
+dist/
+venv/
+setup.py

.pylintrc

@@ -0,0 +1,344 @@
+[MASTER]
+
+# Specify a configuration file.
+#rcfile=
+
+# Python code to execute, usually for sys.path manipulation such as
+# pygtk.require().
+#init-hook=
+
+# Add files or directories to the blacklist. They should be base names, not
+# paths.
+ignore=CVS
+
+# Pickle collected data for later comparisons.
+persistent=yes
+
+# List of plugins (as comma separated values of python modules names) to load,
+# usually to register additional checkers.
+load-plugins=
+
+# Use multiple processes to speed up Pylint.
+jobs=1
+
+# Allow loading of arbitrary C extensions. Extensions are imported into the
+# active Python interpreter and may run arbitrary code.
+unsafe-load-any-extension=no
+
+# A comma-separated list of package or module names from where C extensions may
+# be loaded. Extensions are loading into the active Python interpreter and may
+# run arbitrary code
+extension-pkg-whitelist=
+
+
+[MESSAGES CONTROL]
+
+# Only show warnings with the listed confidence levels. Leave empty to show
+# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
+confidence=
+
+# Enable the message, report, category or checker with the given id(s). You can
+# either give multiple identifier separated by comma (,) or put this option
+# multiple time. See also the "--disable" option for examples.
+#enable=
+
+# Disable the message, report, category or checker with the given id(s). You
+# can either give multiple identifiers separated by comma (,) or put this
+# option multiple times (only on the command line, not in the configuration
+# file where it should appear only once).You can also use "--disable=all" to
+# disable everything first and then reenable specific checks. For example, if
+# you want to run only the similarities checker, you can use "--disable=all
+# --enable=similarities". If you want to run only the classes checker, but have
+# no Warning level messages displayed, use"--disable=all --enable=classes
+# --disable=W"
+
+disable=
+    attribute-defined-outside-init,
+    duplicate-code,
+    fixme,
+    invalid-name,
+    missing-docstring,
+    protected-access,
+    too-few-public-methods,
+
+
+[LOGGING]
+
+# Logging modules to check that the string format arguments are in logging
+# function parameter format
+logging-modules=logging
+
+
+[MISCELLANEOUS]
+
+# List of note tags to take in consideration, separated by a comma.
+notes=FIXME,XXX,TODO
+
+
+[SIMILARITIES]
+
+# Minimum lines number of a similarity.
+min-similarity-lines=4
+
+# Ignore comments when computing similarities.
+ignore-comments=yes
+
+# Ignore docstrings when computing similarities.
+ignore-docstrings=yes
+
+# Ignore imports when computing similarities.
+ignore-imports=no
+
+
+[VARIABLES]
+
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
+
+# A regular expression matching the name of dummy variables (i.e. expectedly
+# not used).
+dummy-variables-rgx=_$|dummy
+
+# List of additional names supposed to be defined in builtins. Remember that
+# you should avoid to define new builtins when possible.
+additional-builtins=
+
+# List of strings which can identify a callback function by name. A callback
+# name must start or end with one of those strings.
+callbacks=cb_,_cb
+
+
+[FORMAT]
+
+# Maximum number of characters on a single line.
+max-line-length=200
+
+# Regexp for a line that is allowed to be longer than the limit.
+ignore-long-lines=^\s*(# )?<?https?://\S+>?$
+
+# Allow the body of an if to be on the same line as the test if there is no
+# else.
+single-line-if-stmt=no
+
+# List of optional constructs for which whitespace checking is disabled
+no-space-check=trailing-comma,dict-separator
+
+# Maximum number of lines in a module
+max-module-lines=2000
+
+# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
+# tab).
+indent-string='    '
+
+# Number of spaces of indent required inside a hanging or continued line.
+indent-after-paren=4
+
+# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
+expected-line-ending-format=
+
+
+[BASIC]
+
+# List of builtins function names that should not be used, separated by a comma
+bad-functions=map,filter,input
+
+# Good variable names which should always be accepted, separated by a comma
+good-names=i,j,k,ex,Run,_
+
+# Bad variable names which should always be refused, separated by a comma
+bad-names=foo,bar,baz,toto,tutu,tata
+
+# Colon-delimited sets of names that determine each other's naming style when
+# the name regexes allow several styles.
+name-group=
+
+# Include a hint for the correct naming format with invalid-name
+include-naming-hint=no
+
+# Regular expression matching correct function names
+function-rgx=[a-z_][a-z0-9_]{2,30}$
+
+# Naming hint for function names
+function-name-hint=[a-z_][a-z0-9_]{2,30}$
+
+# Regular expression matching correct variable names
+variable-rgx=[a-z_][a-z0-9_]{2,30}$
+
+# Naming hint for variable names
+variable-name-hint=[a-z_][a-z0-9_]{2,30}$
+
+# Regular expression matching correct constant names
+const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$
+
+# Naming hint for constant names
+const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$
+
+# Regular expression matching correct attribute names
+attr-rgx=[a-z_][a-z0-9_]{2,30}$
+
+# Naming hint for attribute names
+attr-name-hint=[a-z_][a-z0-9_]{2,30}$
+
+# Regular expression matching correct argument names
+argument-rgx=[a-z_][a-z0-9_]{2,30}$
+
+# Naming hint for argument names
+argument-name-hint=[a-z_][a-z0-9_]{2,30}$
+
+# Regular expression matching correct class attribute names
+class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
+
+# Naming hint for class attribute names
+class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
+
+# Regular expression matching correct inline iteration names
+inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
+
+# Naming hint for inline iteration names
+inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$
+
+# Regular expression matching correct class names
+class-rgx=[A-Z_][a-zA-Z0-9]+$
+
+# Naming hint for class names
+class-name-hint=[A-Z_][a-zA-Z0-9]+$
+
+# Regular expression matching correct module names
+module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
+
+# Naming hint for module names
+module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
+
+# Regular expression matching correct method names
+method-rgx=[a-z_][a-z0-9_]{2,30}$
+
+# Naming hint for method names
+method-name-hint=[a-z_][a-z0-9_]{2,30}$
+
+# Regular expression which should only match function or class names that do
+# not require a docstring.
+no-docstring-rgx=__.*__
+
+# Minimum line length for functions/classes that require docstrings, shorter
+# ones are exempt.
+docstring-min-length=-1
+
+# List of decorators that define properties, such as abc.abstractproperty.
+property-classes=abc.abstractproperty
+
+
+[TYPECHECK]
+
+# Tells whether missing members accessed in mixin class should be ignored. A
+# mixin class is detected if its name ends with "mixin" (case insensitive).
+ignore-mixin-members=yes
+
+# List of module names for which member attributes should not be checked
+# (useful for modules/projects where namespaces are manipulated during runtime
+# and thus existing member attributes cannot be deduced by static analysis
+ignored-modules=
+
+# List of classes names for which member attributes should not be checked
+# (useful for classes with attributes dynamically set).
+ignored-classes=SQLObject, optparse.Values, thread._local, _thread._local
+
+# List of members which are set dynamically and missed by pylint inference
+# system, and so shouldn't trigger E1101 when accessed. Python regular
+# expressions are accepted.
+generated-members=REQUEST,acl_users,aq_parent
+
+# List of decorators that create context managers from functions, such as
+# contextlib.contextmanager.
+contextmanager-decorators=contextlib.contextmanager
+
+
+[SPELLING]
+
+# Spelling dictionary name. Available dictionaries: none. To make it working
+# install python-enchant package.
+spelling-dict=
+
+# List of comma separated words that should not be checked.
+spelling-ignore-words=
+
+# A path to a file that contains private dictionary; one word per line.
+spelling-private-dict-file=
+
+# Tells whether to store unknown words to indicated private dictionary in
+# --spelling-private-dict-file option instead of raising a message.
+spelling-store-unknown-words=no
+
+
+[DESIGN]
+
+# Maximum number of arguments for function / method
+max-args=10
+
+# Argument names that match this expression will be ignored. Default to name
+# with leading underscore
+ignored-argument-names=_.*
+
+# Maximum number of locals for function / method body
+max-locals=25
+
+# Maximum number of return / yield for function / method body
+max-returns=11
+
+# Maximum number of branch for function / method body
+max-branches=26
+
+# Maximum number of statements in function / method body
+max-statements=100
+
+# Maximum number of parents for a class (see R0901).
+max-parents=7
+
+# Maximum number of attributes for a class (see R0902).
+max-attributes=11
+
+# Minimum number of public methods for a class (see R0903).
+min-public-methods=2
+
+# Maximum number of public methods for a class (see R0904).
+max-public-methods=25
+
+
+[CLASSES]
+
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,__new__,setUp
+
+# List of valid names for the first argument in a class method.
+valid-classmethod-first-arg=cls
+
+# List of valid names for the first argument in a metaclass class method.
+valid-metaclass-classmethod-first-arg=mcs
+
+# List of member names, which should be excluded from the protected access
+# warning.
+exclude-protected=_asdict,_fields,_replace,_source,_make
+
+
+[IMPORTS]
+
+# Deprecated modules which should not be used, separated by a comma
+deprecated-modules=regsub,TERMIOS,Bastion,rexec
+
+# Create a graph of every (i.e. internal and external) dependencies in the
+# given file (report RP0402 must not be disabled)
+import-graph=
+
+# Create a graph of external dependencies in the given file (report RP0402 must
+# not be disabled)
+ext-import-graph=
+
+# Create a graph of internal dependencies in the given file (report RP0402 must
+# not be disabled)
+int-import-graph=
+
+
+[EXCEPTIONS]
+
+# Exceptions that will emit a warning when being caught. Defaults to
+# "Exception"
+overgeneral-exceptions=Exception

.travis.yml

@@ -0,0 +1,12 @@
+sudo: false
+language: python
+python:
+  - "2.7"
+  - "3.4"
+  - "3.5"
+  - "3.6"
+  - "3.7-dev"
+install:
+  - pip install tox-travis
+script:
+  - tox

LICENSE

@@ -0,0 +1,13 @@
+Copyright 2018 Dave Gallant
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.rst

@@ -0,0 +1,17 @@
+RFD CLI
+===================
+
+.. image:: https://travis-ci.org/davegallant/rfd_cli.svg?branch=prototype
+    :target: https://travis-ci.org/davegallant/rfd_cli
+
+
+Installation:
+
+    pip install rfd
+
+Usage
+
+    rfd threads 9 --count 10
+
+
+.. image:: images/rfd_threads_9.png

requirements.txt

@@ -0,0 +1,5 @@
+beautifulsoup4==4.6.0
+click==6.7
+colorama==0.3.9
+pytest==3.4.2
+requests==2.18.4

rfd/api.py

@@ -0,0 +1,102 @@
+"""RFD API."""
+
+from math import ceil
+import requests
+from bs4 import BeautifulSoup
+
+try:
+    from urllib.parse import urlparse  # python 3
+except ImportError:
+    from urlparse import urlparse  # python 2
+
+
+def build_web_path(slug):
+    return "https://forums.redflagdeals.com{}".format(slug)
+
+
+def extract_post_id(url):
+    return url.split('/')[3].split('-')[-1]
+
+
+def is_int(number):
+    try:
+        int(number)
+        return True
+    except ValueError:
+        return False
+
+
+def get_vote_score(up_vote, down_vote):
+    return up_vote - down_vote
+
+
+def get_safe_per_page(limit):
+    if limit < 5:
+        return 5
+    elif limit > 40:
+        return 40
+    return limit
+
+
+def users_to_dict(users):
+    users_dict = {}
+    for user in users:
+        users_dict[user.get('user_id')] = user.get('username')
+    return users_dict
+
+
+def strip_html(text):
+    return BeautifulSoup(text, "html.parser").get_text()
+
+
+def is_valid_url(url):
+    result = urlparse(url)
+    return all([result.scheme, result.netloc, result.path])
+
+
+def get_threads(forum_id, limit):
+    threads = []
+    response = requests.get(
+        "https://forums.redflagdeals.com/api/topics?forum_id={}&per_page={}".format(forum_id, get_safe_per_page(limit)))
+    for topic in response.json().get('topics'):
+        threads.append({
+            'title': topic.get('title'),
+            'score': get_vote_score(topic.get('votes').get('total_up'), topic.get('votes').get('total_down')),
+            'url': build_web_path(topic.get('web_path')),
+        })
+    return threads[:limit]
+
+
+def get_posts(post, limit=5):
+    if is_valid_url(post):
+        post_id = extract_post_id(post)
+    elif is_int(post):
+        post_id = post
+    else:
+        raise ValueError()
+
+    if limit == 0:
+        response = requests.get(
+            "https://forums.redflagdeals.com/api/topics/{}/posts?per_page=40&page=1".format(post_id))
+        pages = response.json().get('pager').get('total_pages')
+    elif limit > 40:
+        pages = ceil(limit / 40)
+    else:
+        pages = 1
+
+    # Go through as many pages as necessary
+    for page in range(pages):
+        page = page + 1  # page 0 causes issues
+        response = requests.get(
+            "https://forums.redflagdeals.com/api/topics/{}/posts?per_page={}&page={}".format(post_id,
+                                                                                             get_safe_per_page(
+                                                                                                 limit),
+                                                                                             page))
+
+        users = users_to_dict(response.json().get('users'))
+        for _post in response.json().get('posts'):
+            yield{
+                'body': strip_html(_post.get('body')),
+                'score': get_vote_score(_post.get('votes').get('total_up'), _post.get('votes').get('total_down')),
+                'user': users[_post.get('author_id')],
+            }

rfd/rfd_cli.py

@@ -0,0 +1,91 @@
+import sys
+import os
+import click
+from colorama import init, Fore, Style
+from rfd.api import get_threads, get_posts
+
+init()
+print()
+
+
+def get_terminal_width():
+    _, columns = os.popen('stty size', 'r').read().split()
+    return int(columns)
+
+
+def get_vote_color(score):
+    if score > 0:
+        return Fore.GREEN + " [+" + str(score) + "] "
+    elif score < 0:
+        return Fore.RED + " [" + str(score) + "] "
+    return Fore.BLUE + " [" + str(score) + "] "
+
+
+@click.group()
+def cli():
+    """Welcome to the RFD CLI. (RedFlagDeals.com)"""
+    pass
+
+
+@cli.command(short_help="Displays posts in a specific thread.")
+@click.option('--count', default=5, help='Number of topics. 0 for all topics')
+@click.argument('post_id')
+def posts(count, post_id):
+    """Displays posts in a specific thread.
+
+    post_id can be a full url or post id only
+
+    Example:
+
+    \b
+    url: https://forums.redflagdeals.com/koodo-targeted-public-mobile-12-120-koodo-5gb-40-no-referrals-2173603
+    post_id: 2173603
+    """
+    if count < 0:
+        click.echo("Invalid count.")
+        sys.exit(1)
+
+    try:
+        click.echo("-" * get_terminal_width())
+        for post in get_posts(post_id, count):
+            click.echo(" -" + get_vote_color(post.get('score')) + Fore.RESET +
+                       post.get('body') + Fore.YELLOW + " ({})".format(post.get('user')))
+            click.echo(Style.RESET_ALL)
+            click.echo("-" * get_terminal_width())
+    except ValueError:
+        click.echo("Invalid post id.")
+        sys.exit(1)
+    except AttributeError:
+        click.echo("AttributeError: RFD API did not return expected data.")
+
+
+@cli.command(short_help="Displays threads in the specified forum.")
+@click.option('--count', default=5, help='Number of topics.')
+@click.argument('forum_id')
+def threads(count, forum_id):
+    """Displays threads in the specified forum id.
+
+    Popular forum ids:
+
+    \b
+    9 \t hot deals
+    14 \t computer and electronics
+    15 \t offtopic
+    17 \t entertainment
+    18 \t food and drink
+    40 \t automotive
+    53 \t home and garden
+    67 \t fashion and apparel
+    74 \t shopping discussion
+    88 \t cell phones
+    """
+    _threads = get_threads(forum_id, count)
+    for i, thread in enumerate(_threads, 1):
+        click.echo(" " + str(i) + "." +
+                   get_vote_color(thread.get('score')) + Fore.RESET + thread.get('title'))
+        click.echo(Fore.BLUE + " {}".format(thread.get('url')))
+        click.echo(Style.RESET_ALL)
+
+
+if __name__ == '__main__':
+    cli()

test-requirements.txt

@@ -0,0 +1 @@
+pylint==1.8.2

tests/test_api.py

@@ -0,0 +1,10 @@
+from rfd.api import build_web_path, extract_post_id
+
+
+def test_build_web_path():
+    assert build_web_path("/test") == "https://forums.redflagdeals.com/test"
+
+
+def test_extract_post_id():
+    assert extract_post_id("https://forums.redflagdeals.com/targeted-bob-2173603/120") == '2173603'
+    assert extract_post_id("http://forums.redflagdeals.com/targeted-2173604/120") == '2173604'

tox.ini

@@ -0,0 +1,11 @@
+[tox]
+envlist = py{27,34,35,36,37}
+
+[testenv]
+commands = 
+    py.test
+    pylint rfd
+
+deps = 
+    -r{toxinidir}/requirements.txt
+    -r{toxinidir}/test-requirements.txt