From a83a9cf06413366372d8055be04a0c129bfe72cb Mon Sep 17 00:00:00 2001 From: David Beazley Date: Thu, 28 May 2020 17:41:59 -0500 Subject: [PATCH] Many edits --- .../01_Variable_arguments.md | 55 ++++--- .../02_Anonymous_function.md | 21 ++- .../03_Returning_functions.md | 32 ++-- .../04_Function_decorators.md | 22 ++- .../05_Decorated_methods.md | 90 +++--------- Notes/08_Testing_debugging/00_Overview.md | 2 +- Notes/08_Testing_debugging/01_Testing.md | 87 +++++++---- Notes/08_Testing_debugging/02_Logging.md | 30 ++-- Notes/08_Testing_debugging/03_Debugging.md | 13 +- Notes/09_Packages/01_Packages.md | 87 +++++++---- Notes/09_Packages/02_Third_party.md | 139 +++++++++++++++--- Notes/09_Packages/03_Distribution.md | 87 +++++++++++ Notes/09_Packages/TheEnd.md | 12 ++ Solutions/{7_12 => 7_11}/fileparse.py | 0 Solutions/{7_12 => 7_11}/follow.py | 0 Solutions/{7_12 => 7_11}/pcost.py | 0 Solutions/{7_12 => 7_11}/portfolio.py | 0 Solutions/{7_12 => 7_11}/report.py | 0 Solutions/{7_12 => 7_11}/stock.py | 0 Solutions/{7_12 => 7_11}/tableformat.py | 0 Solutions/{7_12 => 7_11}/ticker.py | 0 Solutions/{7_12 => 7_11}/timethis.py | 0 Solutions/{7_12 => 7_11}/typedproperty.py | 0 23 files changed, 467 insertions(+), 210 deletions(-) create mode 100644 Notes/09_Packages/03_Distribution.md create mode 100644 Notes/09_Packages/TheEnd.md rename Solutions/{7_12 => 7_11}/fileparse.py (100%) rename Solutions/{7_12 => 7_11}/follow.py (100%) rename Solutions/{7_12 => 7_11}/pcost.py (100%) rename Solutions/{7_12 => 7_11}/portfolio.py (100%) rename Solutions/{7_12 => 7_11}/report.py (100%) rename Solutions/{7_12 => 7_11}/stock.py (100%) rename Solutions/{7_12 => 7_11}/tableformat.py (100%) rename Solutions/{7_12 => 7_11}/ticker.py (100%) rename Solutions/{7_12 => 7_11}/timethis.py (100%) rename Solutions/{7_12 => 7_11}/typedproperty.py (100%) diff --git a/Notes/07_Advanced_Topics/01_Variable_arguments.md b/Notes/07_Advanced_Topics/01_Variable_arguments.md index 7ae4d50..d9a2f8c 100644 --- a/Notes/07_Advanced_Topics/01_Variable_arguments.md +++ b/Notes/07_Advanced_Topics/01_Variable_arguments.md @@ -1,25 +1,31 @@ + +[Contents](../Contents) \| [Previous (6.4 Generator Expressions)](../06_Generators/04_More_generators) \| [Next (7.2 Anonymous Functions)](02_Anonymous_function) + # 7.1 Variable Arguments +This section covers variadic function arguments, sometimes described as +`*args` and `**kwargs`. + ### Positional variable arguments (*args) A function that accepts *any number* of arguments is said to use variable arguments. For example: ```python -def foo(x, *args): +def f(x, *args): ... ``` Function call. ```python -foo(1,2,3,4,5) +f(1,2,3,4,5) ``` -The arguments get passed as a tuple. +The extra arguments get passed as a tuple. ```python -def foo(x, *args): +def f(x, *args): # x -> 1 # args -> (2,3,4,5) ``` @@ -30,37 +36,52 @@ A function can also accept any number of keyword arguments. For example: ```python -def foo(x, y, **kwargs): +def f(x, y, **kwargs): ... ``` Function call. ```python -foo(2,3,flag=True,mode='fast',header='debug') +f(2, 3, flag=True, mode='fast', header='debug') ``` The extra keywords are passed in a dictionary. ```python -def foo(x, y, **kwargs): +def f(x, y, **kwargs): # x -> 2 # y -> 3 - # kwargs -> { 'flat': True, 'mode': 'fast', 'header': 'debug' } + # kwargs -> { 'flag': True, 'mode': 'fast', 'header': 'debug' } ``` ### Combining both -A function can also combine any number of variable keyword and non-keyword arguments. -Function definition. +A function can also accept any number of variable keyword and non-keyword arguments. ```python -def foo(*args, **kwargs): +def f(*args, **kwargs): ... ``` -This function takes any combination of positional or keyword arguments. -It is sometimes used when writing wrappers or when you want to pass arguments through to another function. +Function call. + +```python +f(2, 3, flag=True, mode='fast', header='debug') +``` + +The arguments are separated into positional and keyword components + +```python +def f(*args, **kwargs): + # args = (2, 3) + # kwargs -> { 'flag': True, 'mode': 'fast', 'header': 'debug' } + ... +``` + +This function takes any combination of positional or keyword +arguments. It is sometimes used when writing wrappers or when you +want to pass arguments through to another function. ### Passing Tuples and Dicts @@ -68,7 +89,7 @@ Tuples can be expanded into variable arguments. ```python numbers = (2,3,4) -foo(1, *numbers) # Same as f(1,2,3,4) +f(1, *numbers) # Same as f(1,2,3,4) ``` Dictionaries can also be expaded into keyword arguments. @@ -79,12 +100,10 @@ options = { 'delimiter' : ',', 'width' : 400 } -foo(data, **options) -# Same as foo(data, color='red', delimiter=',', width=400) +f(data, **options) +# Same as f(data, color='red', delimiter=',', width=400) ``` -These are not commonly used except when writing library functions. - ## Exercises ### Exercise 7.1: A simple example of variable arguments diff --git a/Notes/07_Advanced_Topics/02_Anonymous_function.md b/Notes/07_Advanced_Topics/02_Anonymous_function.md index ccaa7b9..61cac2f 100644 --- a/Notes/07_Advanced_Topics/02_Anonymous_function.md +++ b/Notes/07_Advanced_Topics/02_Anonymous_function.md @@ -1,3 +1,5 @@ +[Contents](../Contents) \| [Previous (7.1 Variable Arguments)](01_Variable_arguments) \| [Next (7.3 Returning Functions)](03_Returning_functions) + # 7.2 Anonymous Functions and Lambda ### List Sorting Revisited @@ -30,7 +32,9 @@ It seems simple enough. However, how do we sort a list of dicts? By what criteria? -You can guide the sorting by using a *key function*. The *key function* is a function that receives the dictionary and returns the value in a specific key. +You can guide the sorting by using a *key function*. The *key +function* is a function that receives the dictionary and returns the +value of interest for sorting. ```python def stock_name(s): @@ -39,7 +43,7 @@ def stock_name(s): portfolio.sort(key=stock_name) ``` -The value returned by the *key function* determines the sorting. +Here's the result. ```python # Check how the dictionaries are sorted by the `name` key @@ -56,13 +60,16 @@ The value returned by the *key function* determines the sorting. ### Callback Functions -Callback functions are often short one-line functions that are only used for that one operation. For example of previous sorting example. -Programmers often ask for a short-cut, so is there a shorter way to specify custom processing for `sort()`? +In the above example, the key function is an example of a callback +function. The `sort()` method "calls back" to a function you supply. +Callback functions are often short one-line functions that are only +used for that one operation. Programmers often ask for a short-cut +for specifying this extra processing. ### Lambda: Anonymous Functions -Use a lambda instead of creating the function. -In our previous sorting example. +Use a lambda instead of creating the function. In our previous +sorting example. ```python portfolio.sort(key=lambda s: s['name']) @@ -156,6 +163,6 @@ Try sorting the portfolio according to the price of each stock Note: `lambda` is a useful shortcut because it allows you to define a special processing function directly in the call to `sort()` as -opposed to having to define a separate function first (as in part a). +opposed to having to define a separate function first. [Contents](../Contents) \| [Previous (7.1 Variable Arguments)](01_Variable_arguments) \| [Next (7.3 Returning Functions)](03_Returning_functions) diff --git a/Notes/07_Advanced_Topics/03_Returning_functions.md b/Notes/07_Advanced_Topics/03_Returning_functions.md index 1991878..d9155b0 100644 --- a/Notes/07_Advanced_Topics/03_Returning_functions.md +++ b/Notes/07_Advanced_Topics/03_Returning_functions.md @@ -1,6 +1,8 @@ +[Contents](../Contents) \| [Previous (7.2 Anonymous Functions)](02_Anonymous_function) \| [Next (7.4 Decorators)](04_Function_decorators) + # 7.3 Returning Functions -This section introduces the idea of closures. +This section introduces the idea of using functions to create other functions. ### Introduction @@ -27,7 +29,8 @@ Adding 3 4 ### Local Variables -Observe how to inner function refers to variables defined by the outer function. +Observe how to inner function refers to variables defined by the outer +function. ```python def add(x, y): @@ -38,7 +41,8 @@ def add(x, y): return do_add ``` -Further observe that those variables are somehow kept alive after `add()` has finished. +Further observe that those variables are somehow kept alive after +`add()` has finished. ```python >>> a = add(3,4) @@ -51,7 +55,7 @@ Adding 3 4 # Where are these values coming from? ### Closures -When an inner function is returned as a result, the inner function is known as a *closure*. +When an inner function is returned as a result, that inner function is known as a *closure*. ```python def add(x, y): @@ -62,7 +66,10 @@ def add(x, y): return do_add ``` -*Essential feature: A closure retains the values of all variables needed for the function to run properly later on.* +*Essential feature: A closure retains the values of all variables + needed for the function to run properly later on.* Think of a +closure as a function plus an extra environment that holds the values +of variables that it depends on. ### Using Closures @@ -99,7 +106,7 @@ Closures carry extra information around. ```python def add(x, y): def do_add(): - print('Adding %s + %s -> %s' % (x, y, x + y)) + print(f'Adding {x} + {y} -> {x+y}') return do_add def after(seconds, func): @@ -110,8 +117,6 @@ after(30, add(2, 3)) # `do_add` has the references x -> 2 and y -> 3 ``` -A function can have its own little environment. - ### Code Repetition Closures can also be used as technique for avoiding excessive code repetition. @@ -122,11 +127,12 @@ You can write functions that make code. ### Exercise 7.7: Using Closures to Avoid Repetition One of the more powerful features of closures is their use in -generating repetitive code. If you refer back to exercise 5.2 -recall the code for defining a property with type checking. +generating repetitive code. If you refer back to [Exercise +5.7](../05_Object_model/02_Classes_encapsulation), recall the code for +defining a property with type checking. ```python -class Stock(object): +class Stock: def __init__(self, name, shares, price): self.name = name self.shares = shares @@ -173,7 +179,7 @@ Now, try it out by defining a class like this: ```python from typedproperty import typedproperty -class Stock(object): +class Stock: name = typedproperty('name', str) shares = typedproperty('shares', int) price = typedproperty('price', float) @@ -211,7 +217,7 @@ Float = lambda name: typedproperty(name, float) Now, rewrite the `Stock` class to use these functions instead: ```python -class Stock(object): +class Stock: name = String('name') shares = Integer('shares') price = Float('price') diff --git a/Notes/07_Advanced_Topics/04_Function_decorators.md b/Notes/07_Advanced_Topics/04_Function_decorators.md index 1966976..4ad922d 100644 --- a/Notes/07_Advanced_Topics/04_Function_decorators.md +++ b/Notes/07_Advanced_Topics/04_Function_decorators.md @@ -1,3 +1,5 @@ +[Contents](../Contents) \| [Previous (7.3 Returning Functions)](03_Returning_functions) \| [Next (7.5 Decorated Methods)](05_Decorated_methods) + # 7.4 Function Decorators This section introduces the concept of a decorator. This is an advanced @@ -12,7 +14,7 @@ def add(x, y): return x + y ``` -Now, consider the function with some logging. +Now, consider the function with some logging added to it. ```python def add(x, y): @@ -32,13 +34,15 @@ def sub(x, y): *Observation: It's kind of repetitive.* -Writing programs where there is a lot of code replication is often really annoying. -They are tedious to write and hard to maintain. -Especially if you decide that you want to change how it works (i.e., a different kind of logging perhaps). +Writing programs where there is a lot of code replication is often +really annoying. They are tedious to write and hard to maintain. +Especially if you decide that you want to change how it works (i.e., a +different kind of logging perhaps). -### Example continuation +### Code that makes logging -Perhaps you can make *logging wrappers*. +Perhaps you can make a function that makes functions with logging +added to them. A wrapper. ```python def logged(func): @@ -65,7 +69,9 @@ logged_add(3, 4) # You see the logging message appear This example illustrates the process of creating a so-called *wrapper function*. -**A wrapper is a function that wraps another function with some extra bits of processing.** +A wrapper is a function that wraps around another function with some +extra bits of processing, but otherwise works in the exact same way +as the original function. ```python >>> logged_add(3, 4) @@ -100,6 +106,8 @@ It is said to *decorate* the function. There are many more subtle details to decorators than what has been presented here. For example, using them in classes. Or using multiple decorators with a function. However, the previous example is a good illustration of how their use tends to arise. +Usually, it's in response to repetitive code appearing across a wide range of +function definitions. A decorator can move that code to a central definition. ## Exercises diff --git a/Notes/07_Advanced_Topics/05_Decorated_methods.md b/Notes/07_Advanced_Topics/05_Decorated_methods.md index 671f536..73e7692 100644 --- a/Notes/07_Advanced_Topics/05_Decorated_methods.md +++ b/Notes/07_Advanced_Topics/05_Decorated_methods.md @@ -1,6 +1,8 @@ +[Contents](../Contents) \| [Previous (7.4 Decorators)](04_Function_decorators) \| [Next (8 Testing and Debugging)](../08_Testing_debugging/00_Overview) + # 7.5 Decorated Methods -This section discusses a few common decorators that are used in +This section discusses a few built-in decorators that are used in combination with method definitions. ### Predefined Decorators @@ -8,7 +10,7 @@ combination with method definitions. There are predefined decorators used to specify special kinds of methods in class definitions. ```python -class Foo(object): +class Foo: def bar(self,a): ... @@ -29,8 +31,9 @@ Let's go one by one. ### Static Methods -`@staticmethod` is used to define a so-called *static* class methods (from C++/Java). -A static method is a function that is part of the class, but which does *not* operate on instances. +`@staticmethod` is used to define a so-called *static* class methods +(from C++/Java). A static method is a function that is part of the +class, but which does *not* operate on instances. ```python class Foo(object): @@ -42,17 +45,19 @@ class Foo(object): >>> ``` -Static methods are sometimes used to implement internal supporting code for a class. -For example, code to help manage created instances (memory management, system resources, persistence, locking, etc). +Static methods are sometimes used to implement internal supporting +code for a class. For example, code to help manage created instances +(memory management, system resources, persistence, locking, etc). They're also used by certain design patterns (not discussed here). ### Class Methods -`@classmethod` is used to define class methods. -A class method is a method that receives the *class* object as the first parameter instead of the instance. +`@classmethod` is used to define class methods. A class method is a +method that receives the *class* object as the first parameter instead +of the instance. ```python -class Foo(object): +class Foo: def bar(self): print(self) @@ -71,7 +76,7 @@ class Foo(object): Class methods are most often used as a tool for defining alternate constructors. ```python -class Date(object): +class Date: def __init__(self,year,month,day): self.year = year self.month = month @@ -90,7 +95,7 @@ d = Date.today() Class methods solve some tricky problems with features like inheritance. ```python -class Date(object): +class Date: ... @classmethod def today(cls): @@ -106,62 +111,7 @@ d = NewDate.today() ## Exercises -Start this exercise by defining a `Date` class. For example: - -``` ->>> class Date(object): - def __init__(self,year,month,day): - self.year = year - self.month = month - self.day = day - ->>> d = Date(2010, 4, 13) ->>> d.year, d.month, d.day -(2010, 4, 13) ->>> -``` - -### Exercise 7.11: Class Methods - -A common use of class methods is to provide alternate constructors -(epecially since Python doesn't support overloaded methods). Modify -the `Date` class to have a class method `today()` that creates a date -from today's date. - -```python ->>> import time ->>> class Date(object): - def __init__(self,year,month,day): - self.year = year - self.month = month - self.day = day - @classmethod - def today(cls): - t = time.localtime() - return cls(t.tm_year, t.tm_mon, t.tm_mday) - ->>> d = Date.today() ->>> d.year, d.month, d.day -... output varies. Should be today ... ->>> -``` - -One reason you should use class methods for this is that they work -with inheritance. For example, try this: - -```python ->>> class CustomDate(Date): - def yow(self): - print('Yow!') - ->>> d = CustomDate.today() -<__main__.CustomDate object at 0x10923d400> ->>> d.yow() -Yow! ->>> -``` - -### Exercise 7.12: Class Methods in Practice +### Exercise 7.11: Class Methods in Practice In your `report.py` and `portfolio.py` files, the creation of a `Portfolio` object is a bit muddled. For example, the `report.py` program has code like this: @@ -186,7 +136,7 @@ and the `portfolio.py` file defines `Portfolio()` with an odd initializer like this: ```python -class Portfolio(object): +class Portfolio: def __init__(self, holdings): self.holdings = holdings ... @@ -202,7 +152,7 @@ Like this: import stock -class Portfolio(object): +class Portfolio: def __init__(self): self.holdings = [] @@ -222,7 +172,7 @@ class method for it: import fileparse import stock -class Portfolio(object): +class Portfolio: def __init__(self): self.holdings = [] diff --git a/Notes/08_Testing_debugging/00_Overview.md b/Notes/08_Testing_debugging/00_Overview.md index 679e2d7..8652733 100644 --- a/Notes/08_Testing_debugging/00_Overview.md +++ b/Notes/08_Testing_debugging/00_Overview.md @@ -1,7 +1,7 @@ # 8. Overview This section introduces a few basic topics related to testing, -logging, and debugging. +logging, and debugging. * [8.1 Testing](01_Testing) * [8.2 Logging, error handling and diagnostics](02_Logging) diff --git a/Notes/08_Testing_debugging/01_Testing.md b/Notes/08_Testing_debugging/01_Testing.md index 503de4a..df3a942 100644 --- a/Notes/08_Testing_debugging/01_Testing.md +++ b/Notes/08_Testing_debugging/01_Testing.md @@ -1,14 +1,18 @@ +[Contents](../Contents) \| [Previous (7.5 Decorated Methods)](../07_Advanced_Topics/05_Decorated_methods) \| [Next (8.2 Logging)](02_Logging) + # 8.1 Testing ## Testing Rocks, Debugging Sucks -The dynamic nature of Python makes testing critically important to most applications. -There is no compiler to find your bugs. The only way to find bugs is to run the code and make sure you try out all of its features. +The dynamic nature of Python makes testing critically important to +most applications. There is no compiler to find your bugs. The only +way to find bugs is to run the code and make sure you try out all of +its features. ## Assertions -The assertion statement is an internal check for the program. -If an expression is not true, it raises a `AssertionError` exception. +The `assert` statement is an internal check for the program. If an +expression is not true, it raises a `AssertionError` exception. `assert` statement syntax. @@ -22,15 +26,18 @@ For example. assert isinstance(10, int), 'Expected int' ``` -It shouldn't be used to check the user-input. +It shouldn't be used to check the user-input (i.e., data entered +on a web form or something). It's purpose is more for internal +checks and invariants (conditions that should always be true). ### Contract Programming -Also known as Design By Contract, liberal use of assertions is an approach for designing -software. It prescribes that software designers should define precise -interface specifications for the components of the software. +Also known as Design By Contract, liberal use of assertions is an +approach for designing software. It prescribes that software designers +should define precise interface specifications for the components of +the software. -For example, you might put assertions on all inputs and outputs. +For example, you might put assertions on all inputs of a function. ```python def add(x, y): @@ -39,7 +46,8 @@ def add(x, y): return x + y ``` -Checking inputs will immediately catch callers who aren't using appropriate arguments. +Checking inputs will immediately catch callers who aren't using +appropriate arguments. ```python >>> add(2, 3) @@ -64,9 +72,12 @@ assert add(2,2) == 4 This way you are including the test in the same module as your code. -*Benefit: If the code is obviously broken, attempts to import the module will crash.* +*Benefit: If the code is obviously broken, attempts to import the + module will crash.* -This is not recommended for exhaustive testing. +This is not recommended for exhaustive testing. It's more of a +basic "smoke test". Does the function work on any example at all? +If not, then something is definitely broken. ### `unittest` Module @@ -79,10 +90,10 @@ def add(x, y): return x + y ``` -You can create a separate testing file. For example: +Now, suppose you want to test it. Create a separate testing file like this. ```python -# testsimple.py +# test_simple.py import simple import unittest @@ -91,7 +102,7 @@ import unittest Then define a testing class. ```python -# testsimple.py +# test_simple.py import simple import unittest @@ -106,7 +117,7 @@ The testing class must inherit from `unittest.TestCase`. In the testing class, you define the testing methods. ```python -# testsimple.py +# test_simple.py import simple import unittest @@ -146,14 +157,15 @@ self.assertAlmostEqual(x,y,places) self.assertRaises(exc, callable, arg1, arg2, ...) ``` -This is not an exhaustive list. There are other assertions in the module. +This is not an exhaustive list. There are other assertions in the +module. ### Running `unittest` To run the tests, turn the code into a script. ```python -# testsimple.py +# test_simple.py ... @@ -164,7 +176,7 @@ if __name__ == '__main__': Then run Python on the test file. ```bash -bash % python3 testsimple.py +bash % python3 test_simple.py F. ======================================================== FAIL: test_simple (__main__.TestAdd) @@ -180,7 +192,8 @@ FAILED (failures=1) ### Commentary -Effective unit testing is an art and it can grow to be quite complicated for large applications. +Effective unit testing is an art and it can grow to be quite +complicated for large applications. The `unittest` module has a huge number of options related to test runners, collection of results and other aspects of testing. Consult @@ -188,23 +201,39 @@ the documentation for details. ### Third Party Test Tools -We won't cover any third party test tools in this course. +The built-in `unittest` module has the advantage of being available everywhere--it's +part of Python. However, many programmers also find it to be quite verbose. +A popular alternative is [pytest](https://docs.pytest.org/en/latest/). With pytest, +your testing file simplifies to something like the following: -However, there are a few popular alternatives and complements to -`unittest`. +```python +# test_simple.py +import simple -* [pytest](https://pytest.org) - A popular alternative. -* [coverage](http://coverage.readthedocs.io) - Code coverage. +def test_simple(): + assert simple.add(2,2) == 4 + +def test_str(): + assert simple.add('hello','world') == 'helloworld' +``` + +To run the tests, you simply type a command such as `python -m pytest`. It will +discover all of the tests and run them. + +There's a lot more to `pytest` than this example, but it's usually pretty easy to +get started should you decide to try it out. ## Exercises In this exercise, you will explore the basic mechanics of using Python's `unittest` module. -In earlier exercises, you wrote a file `stock.py` that contained a `Stock` -class. For this exercise, it assumed that you're using the code written -for Exercise 7.3. If, for some reason, that's not working, -you might want to copy the solution from `Solutions/7_3` to your working +In earlier exercises, you wrote a file `stock.py` that contained a +`Stock` class. For this exercise, it assumed that you're using the +code written for [Exercise +7.9](../07_Advanced_Topics/03_Returning_functions) involving +typed-properties. If, for some reason, that's not working, you might +want to copy the solution from `Solutions/7_9` to your working directory. ### Exercise 8.1: Writing Unit Tests diff --git a/Notes/08_Testing_debugging/02_Logging.md b/Notes/08_Testing_debugging/02_Logging.md index 4a28d6d..973bb7a 100644 --- a/Notes/08_Testing_debugging/02_Logging.md +++ b/Notes/08_Testing_debugging/02_Logging.md @@ -1,16 +1,20 @@ +[Contents](../Contents) \| [Previous (8.1 Testing)](01_Testing) \| [Next (8.3 Debugging)](03_Debugging) + # 8.2 Logging This section briefly introduces the logging module. ### `logging` Module -The `logging` module is a standard library module for recording diagnostic information. -It's also a very large module with a lot of sophisticated functionality. -We will show a simple example to illustrate its usefulness. +The `logging` module is a standard library module for recording +diagnostic information. It's also a very large module with a lot of +sophisticated functionality. We will show a simple example to +illustrate its usefulness. ### Exceptions Revisited -In the exercises, we wrote a function `parse()` that looked something like this: +In the exercises, we wrote a function `parse()` that looked something +like this: ```python # fileparse.py @@ -91,7 +95,7 @@ log.debug(message [, args]) *Each method represents a different level of severity.* -All of them create a formatted log message. `args` is used for the `%` operator. +All of them create a formatted log message. `args` is used with the `%` operator to create the message. ```python logmsg = message % args # Written to the log @@ -114,21 +118,21 @@ if __name__ == '__main__': ) ``` -Typically, this is a one-time configuration at program startup. -The configuration is separate from the code that makes the logging calls. +Typically, this is a one-time configuration at program startup. The +configuration is separate from the code that makes the logging calls. ### Comments -Logging is highly configurable. -You can adjust every aspect of it: output files, levels, message formats, etc. -However, the code that uses logging doesn't have to worry about that. +Logging is highly configurable. You can adjust every aspect of it: +output files, levels, message formats, etc. However, the code that +uses logging doesn't have to worry about that. ## Exercises ### Exercise 8.2: Adding logging to a module -In Exercise 3.3, you added some error handling to the -`fileparse.parse_csv()` function. It looked like this: +In `fileparse.py`, there is some error handling related to +exceptions caused by bad input. It looks like this: ```python # fileparse.py @@ -300,6 +304,6 @@ logging.basicConfig( ``` Again, you'd need to put this someplace in the startup steps of your -program. +program. For example, where would you put this in your `report.py` program? [Contents](../Contents) \| [Previous (8.1 Testing)](01_Testing) \| [Next (8.3 Debugging)](03_Debugging) diff --git a/Notes/08_Testing_debugging/03_Debugging.md b/Notes/08_Testing_debugging/03_Debugging.md index 04a7fb5..921ffe3 100644 --- a/Notes/08_Testing_debugging/03_Debugging.md +++ b/Notes/08_Testing_debugging/03_Debugging.md @@ -1,3 +1,5 @@ +[Contents](../Contents) \| [Previous (8.2 Logging)](02_Logging) \| [Next (9 Packages)](../09_Packages/00_Overview) + # 8.3 Debugging ### Debugging Tips @@ -62,7 +64,8 @@ AttributeError: 'int' object has no attribute 'append' >>> ``` -It preserves the interpreter state. That means that you can go poking around after the crash. Checking variable values and other state. +It preserves the interpreter state. That means that you can go poking +around after the crash. Checking variable values and other state. ### Debugging with Print @@ -102,7 +105,9 @@ def some_function(): ``` This starts the debugger at the `breakpoint()` call. -For earlier Python versions: + +In earlier Python versions, you did this. You'll sometimes see this +mentioned in other debugging guides. ```python import pdb @@ -119,7 +124,9 @@ You can also run an entire program under debugger. bash % python3 -m pdb someprogram.py ``` -It will automatically enter the debugger before the first statement. Allowing you to set breakpoints and change the configuration. +It will automatically enter the debugger before the first +statement. Allowing you to set breakpoints and change the +configuration. Common debugger commands: diff --git a/Notes/09_Packages/01_Packages.md b/Notes/09_Packages/01_Packages.md index 2fe3563..4e90120 100644 --- a/Notes/09_Packages/01_Packages.md +++ b/Notes/09_Packages/01_Packages.md @@ -1,6 +1,10 @@ +[Contents](../Contents) \| [Previous (8.3 Debugging)](../08_Testing_debugging/03_Debugging) \| [Next (9.2 Third Party Packages)](02_Third_party) + # 9.1 Packages -This section introduces the concept of a package. +If writing a larger program, you don't really want to organize it as a +large of collection of standalone files at the top level. This +section introduces the concept of a package. ### Modules @@ -27,7 +31,8 @@ b = foo.spam('Hello') ### Packages vs Modules -For larger collections of code, it is common to organize modules into a package. +For larger collections of code, it is common to organize modules into +a package. ```code # From this @@ -43,17 +48,18 @@ porty/ fileparse.py ``` -You pick a name and make a top-level directory. `porty` in the example above. +You pick a name and make a top-level directory. `porty` in the example +above (clearly picking this name is the most important first step). -Add an `__init__.py` file. It may be empty. +Add an `__init__.py` file to the directory. It may be empty. -Put your source files into it. +Put your source files into the directory. ### Using a Package A package serves as a namespace for imports. -This means that there are multilevel imports. +This means that there are now multilevel imports. ```python import porty.report @@ -64,25 +70,25 @@ There are other variations of import statements. ```python from porty import report -port = report.read_portfolio('port.csv') +port = report.read_portfolio('portfolio.csv') from porty.report import read_portfolio -port = read_portfolio('port.csv') +port = read_portfolio('portfolio.csv') ``` ### Two problems There are two main problems with this approach. -* imports between files in the same package. -* Main scripts placed inside the package. +* imports between files in the same package break. +* Main scripts placed inside the package break. -Both break. +So, basically everything breaks. But, other than that, it works. ### Problem: Imports -Imports between files in the same package *must include the package name in the import*. -Remember the structure. +Imports between files in the same package *must now include the +package name in the import*. Remember the structure. ```code porty/ @@ -92,7 +98,7 @@ porty/ fileparse.py ``` -Import example. +Modified import example. ```python # report.py @@ -113,7 +119,8 @@ import fileparse # BREAKS. fileparse not found ### Relative Imports -However, you can use `.` to refer to the current package. Instead of the package name. +Instead of directly using the package name, +you can use `.` to refer to the current package. ```python # report.py @@ -133,16 +140,24 @@ This makes it easy to rename the package. ### Problem: Main Scripts -Running a submodule as a main script breaks. +Running a package submodule as a main script breaks. ```bash bash $ python porty/pcost.py # BREAKS ... ``` -*Reason: You are running Python on a single file and Python doesn't see the rest of the package structure correctly (`sys.path` is wrong).* +*Reason: You are running Python on a single file and Python doesn't + see the rest of the package structure correctly (`sys.path` is + wrong).* -All imports break. +All imports break. To fix this, you need to run your program in +a different way, using the `-m` option. + +```bash +bash $ python -m porty.pcost # WORKS +... +``` ### `__init__.py` files @@ -156,7 +171,7 @@ from .pcost import portfolio_cost from .report import portfolio_report ``` -Makes names appear at the *top-level* when importing. +This makes names appear at the *top-level* when importing. ```python from porty import portfolio_cost @@ -170,16 +185,15 @@ from porty import pcost pcost.portfolio_cost('portfolio.csv') ``` -### Solution for scripts +### Another solution for scripts -Use `-m package.module` option. +As noted, you now need to use `-m package.module` to +run scripts within your package. ```bash bash % python3 -m porty.pcost portfolio.csv ``` -It will run the code in a proper package environment. - There is another alternative: Write a new top-level script. ```python @@ -190,13 +204,24 @@ import sys porty.pcost.main(sys.argv) ``` -This script lives *outside* the package. +This script lives *outside* the package. For example, looking at the directory +structure: + +``` +pcost.py # top-level-script +porty/ # package directory + __init__.py + pcost.py + ... +``` ### Application Structure -Code organization and file structure is key to the maintainability of an application. +Code organization and file structure is key to the maintainability of +an application. -One recommended structure is the following. +There is no "one-size fits all" approach for Python. However, one +structure that works for a lot of problems is something like this. ```code porty-app/ @@ -210,11 +235,15 @@ porty-app/ fileparse.py ``` -Top-level scripts need to exist outside the code package. One level up. +The top-level `porty-app` is a container for everything else--documentation, +top-level scripts, examples, etc. + +Again, top-level scripts (if any) need to exist outside the code +package. One level up. ```python #!/usr/bin/env python3 -# script.py +# porty-add/script.py import sys import porty @@ -306,7 +335,7 @@ scripts, and other things. These files need to exist OUTSIDE of the `porty/` directory you made above. Create a new directory called `porty-app`. Move the `porty` directory -you created in part (a) into that directory. Copy the +you created in Exercise 9.1 into that directory. Copy the `Data/portfolio.csv` and `Data/prices.csv` test files into this directory. Additionally create a `README.txt` file with some information about yourself. Your code should now be organized as diff --git a/Notes/09_Packages/02_Third_party.md b/Notes/09_Packages/02_Third_party.md index e158204..8eff73a 100644 --- a/Notes/09_Packages/02_Third_party.md +++ b/Notes/09_Packages/02_Third_party.md @@ -1,45 +1,144 @@ -# 9.2 Third Party Modules +[Contents](../Contents) \| [Previous (9.1 Packages)](01_Packages) \| [Next (9.3 Distribution)](03_Distribution) -### Introduction +# 9.2 Third Party Modules Python has a large library of built-in modules (*batteries included*). -There are even more third party modules. Check them in the [Python Package Index](https://pypi.org/) or PyPi. Or just do a Google search for a topic. +There are even more third party modules. Check them in the [Python Package Index](https://pypi.org/) or PyPi. +Or just do a Google search for a specific topic. -### Some Notable Modules +How to handle third-party dependencies is an ever-evolving topic with +Python. This section merely covers the basics to help you wrap +your brain around how it works. -* `requests`: Accessing web services. -* `numpy`, `scipy`: Arrays and vector mathematics. -* `pandas`: Stats and data analysis. -* `django`, `flask`: Web programming. -* `sqlalchemy`: Databases and ORM. -* `ipython`: Alternative interactive shell. +### The Module Search Path + +`sys.path` is a directory that contains the list of all directories +checked by the `import` statement. Look at it: + +```python +>>> import sys +>>> sys.path +... look at the result ... +>>> +``` + +If you import something and it's not located in one of those +directories, you will get an `ImportError` exception. + +### Standard Library Modules + +Modules from Python's standard library usually come from a location +such as `/usr/local/lib/python3.6'. You can find out for certain +by trying a short test: + +```python +>>> import re +>>> re + +>>> +``` + +Simply looking at a module in the REPL is a good debugging tip +to know about. It will show you the location of the file. + +### Third-party Modules + +Third party modules are usually located in a dedicated +`site-packages` directory. You'll see it if you perform +the same steps as above: + +```python +>>> import numpy + +>>> +``` + +Again, looking at a module is a good debugging tip if you're +trying to figure out why something related to `import` isn't working +as expected. ### Installing Modules -Most common classic technique: `pip`. +The most common technique for installing a third-party module is to use +`pip`. For example: ```bash bash % python3 -m pip install packagename ``` -This command will download the package and install it globally in your Python folder. Somewhere like: - -```code -/usr/local/lib/python3.6/site-packages -``` +This command will download the package and install it in the `site-packages` +directory. ### Problems * You may be using an installation of Python that you don't directly control. * A corporate approved installation - * The Python version that comes with the OS. + * You're using the Python version that comes with the OS. * You might not have permission to install global packages in the computer. -* Your program might have unusual dependencies. +* There might be other dependencies. -### Talk about environments... +### Virtual Environments +A common solution to package installation issues is to create a +so-called "virtual environment" for yourself. Naturally, there is no +"one way" to do this--in fact, there are several competing tools and +techniques. However, if you are using a standard Python installation, +you can try typing this: + +```bash +bash % python -m venv mypython +bash % +``` + +After a few moments of waiting, you will have a new directory +`mypython` that's your own little Python install. Within that +directory you'll find a `bin/` directory (Unix) or a `Scripts/` +directory (Windows). If you run the `activate` script found there, it +will "activate" this version of Python, making it the default `python` +command for the shell. For example: + +```bash +bash % source mypython/bin/activate +(mypython) bash % +``` + +From here, you can now start installing Python packages for yourself. +For example: + +``` +(mypython) bash % python -m pip install pandas +... +``` + +For the purposes of experimenting and trying out different +packages, a virtual environment will usually work fine. If, +on the other hand, you're creating an application and it +has specific package dependencies, that is a slightly +different problem. + +### Handling Third-Party Dependencies in Your Application + +If you have written an application and it has specific third-party +dependencies, one challange concerns the creation and preservation of +the environment that includes your code and the dependencies. Sadly, +this has been an area of great confusion and frequent change over +Python's lifetime. It continues to evolve even now. + +Rather than provide information that's bound to be out of date soon, +I refer you to the [Python Packaging User Guide](https://packaging.python.org). ## Exercises -(rewrite) +### Exercise 9.4 : Creating a Virtual Environment + +See if you can recreate the steps of making a virtual environment and installing +pandas into it as shown above. + +[Contents](../Contents) \| [Previous (9.1 Packages)](01_Packages) \| [Next (9.3 Distribution)](03_Distribution) + + + + + + diff --git a/Notes/09_Packages/03_Distribution.md b/Notes/09_Packages/03_Distribution.md new file mode 100644 index 0000000..e7848b1 --- /dev/null +++ b/Notes/09_Packages/03_Distribution.md @@ -0,0 +1,87 @@ +[Contents](../Contents) \| [Previous (9.2 Third Party Packages)](02_Third_party) \| [Next (The End)](TheEnd) + +# 9.3 Distribution + +At some point you might want to give your code to someone else, possibly just a co-worker. +This section gives the most basic technique of doing that. For more detailed +information, you'll need to consult the [Python Packaging User Guide](https://packaging.python.org). + +### Creating a setup.py file + +Add a `setup.py` file to the top-level of your project directory. + +```python +# setup.py +import setuptools + +setuptools.setup( + name="porty", + version="0.0.1", + author="Your Name", + author_email="you@example.com", + description="Practical Python Code", + packages=setuptools.find_packages(), +) +``` + +### Creating MANIFEST.in + +If there are additional files associated with your project, specify them with a `MANIFEST.in` file. +For example: + +``` +# MANIFEST.in +include *.csv +``` + +Put the `MANIFEST.in` file in the same directory as `setup.py`. + +### Creating a source distribution + +To create a distribution of your code, use the `setup.py` file. For example: + +``` +bash % python setup.py sdist +``` + +This will create a `.tar.gz` or `.zip` file in the directory `dist/`. That file is something +that you can now give away to others. + +### Installing your code + +Others can install your Python code using `pip` in the same way that they do for other +packages. They simply need to supply the file created in the previous step. +For example: + +``` +bash % python -m pip install porty-0.0.1.tar.gz +``` + +### Commentary + +The steps above describe the absolute most minimal basics of creating +a package of Python code that you can give to another person. In +reality, it can be much more complicated depending on third-party +dependencies, whether or not your application includes foreign code +(i.e., C/C++), and so forth. Covering that is outside the scope of +this course. We've only taken a tiny first step. + +## Exercises + +### Exercise 9.5: Make a package + +Take the `porty-app/` code you created for Exercise 9.3 and see if you +can recreate the steps described here. Specifically, add a `setup.py` +file and a `MANIFEST.in` file to the top-level directory. +Create a source distribution file by running `python setup.py sdist`. + +As a final step, see if you can install your package into a Python +virtual environment. + +[Contents](../Contents) \| [Previous (9.2 Third Party Packages)](02_Third_party) \| [Next (The End)](TheEnd) + + + + + + diff --git a/Notes/09_Packages/TheEnd.md b/Notes/09_Packages/TheEnd.md new file mode 100644 index 0000000..96b357c --- /dev/null +++ b/Notes/09_Packages/TheEnd.md @@ -0,0 +1,12 @@ +# The End! + +You've made it to the end of the course. Thanks for your time and your attention. +May your future Python hacking be fun and productive! + +I'm always happy to get feedback. You can find me at [https://dabeaz.com](https://dabeaz.com) +or on Twitter at [@dabeaz](https://twitter.com/dabeaz). + +- David Beazley + +[Contents](../Contents) \| [Home](../..) + diff --git a/Solutions/7_12/fileparse.py b/Solutions/7_11/fileparse.py similarity index 100% rename from Solutions/7_12/fileparse.py rename to Solutions/7_11/fileparse.py diff --git a/Solutions/7_12/follow.py b/Solutions/7_11/follow.py similarity index 100% rename from Solutions/7_12/follow.py rename to Solutions/7_11/follow.py diff --git a/Solutions/7_12/pcost.py b/Solutions/7_11/pcost.py similarity index 100% rename from Solutions/7_12/pcost.py rename to Solutions/7_11/pcost.py diff --git a/Solutions/7_12/portfolio.py b/Solutions/7_11/portfolio.py similarity index 100% rename from Solutions/7_12/portfolio.py rename to Solutions/7_11/portfolio.py diff --git a/Solutions/7_12/report.py b/Solutions/7_11/report.py similarity index 100% rename from Solutions/7_12/report.py rename to Solutions/7_11/report.py diff --git a/Solutions/7_12/stock.py b/Solutions/7_11/stock.py similarity index 100% rename from Solutions/7_12/stock.py rename to Solutions/7_11/stock.py diff --git a/Solutions/7_12/tableformat.py b/Solutions/7_11/tableformat.py similarity index 100% rename from Solutions/7_12/tableformat.py rename to Solutions/7_11/tableformat.py diff --git a/Solutions/7_12/ticker.py b/Solutions/7_11/ticker.py similarity index 100% rename from Solutions/7_12/ticker.py rename to Solutions/7_11/ticker.py diff --git a/Solutions/7_12/timethis.py b/Solutions/7_11/timethis.py similarity index 100% rename from Solutions/7_12/timethis.py rename to Solutions/7_11/timethis.py diff --git a/Solutions/7_12/typedproperty.py b/Solutions/7_11/typedproperty.py similarity index 100% rename from Solutions/7_12/typedproperty.py rename to Solutions/7_11/typedproperty.py