>>> def my_decorator(func):
...     def wrapper(*args, **kwargs):
...         print("Before call")
...         result = func(*args, **kwargs)
...         print("After call")
...         return result
...     return wrapper
...
>>> @my_decorator
... def add(a, b):
...     "Our add function"
...     return a + b
...
>>> add(1, 3)
Before call
After call
4

>>> def add_number(num):
...     def adder(number):
...         'adder is a closure'
...         return num + number
...     return adder
...
>>> a_10 = add_number(10)
>>> a_10(21)
31
>>> a_10(34)
44
>>> a_5 = add_number(5)
>>> a_5(3)
8

>>> from collections import Iterable
>>> l = [1, 2, 3, 4]
>>> isinstance(l, Iterable)
True

>>> sum(i*i for i in range(10))         # sum of squares 0, 1, 4, ... 81
285

text = '{'a':1}'
eval(text) # will turn text from a string object into a dictionary object

>>> int('5')
5
>>> float('5.0')
5.0
>>> float('5')
5.0
>>> int(5.0)
5
>>> float(5)
5.0
>>> int('5.0')
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
ValueError: invalid literal for int() with base 10: '5.0'
>>> int(float('5.0'))
5

>>> ["foo", "bar", "baz"].index("bar")
1

for idx, val in enumerate(list):
    print(idx, val)

generator = (word + '!' for word in 'baby let me iterate ya'.split())
# The generator object is now created, ready to be iterated over.
# No exclamation marks added yet at this point.

for val in generator: # real processing happens here, during iteration
    print val,
baby! let! me! iterate! ya!

for val in generator:
    print val,
# Nothing printed! No more data, generator stream already exhausted above.

class BeyonceIterable(object):
    def __iter__(self):
        """
        The iterable interface: return an iterator from __iter__().

        Every generator is an iterator implicitly (but not vice versa!),
        so implementing `__iter__` as a generator is the easiest way
        to create streamed iterables.

        """
        for word in 'baby let me iterate ya'.split():
            yield word + '!'  # uses yield => __iter__ is a generator

iterable = BeyonceIterable()

for val in iterable:  # iterator created here
    print val,
baby! let! me! iterate! ya!

for val in iterable:  # another iterator created here
    print val,
baby! let! me! iterate! ya!

# define a list
my_list = [4, 7, 0, 3]

# get an iterator using iter()
my_iter = iter(my_list)

## iterate through it using next()

#prints 4
print(next(my_iter))

#prints 7
print(next(my_iter))

## next(obj) is same as obj.__next__()

#prints 0
print(my_iter.__next__())

#prints 3
print(my_iter.__next__())

## This will raise error, no items left
next(my_iter)

for element in my_list:
    print(element)

for element in iterable:
    # do something with element
# Is actually implemented as.

# create an iterator object from that iterable
iter_obj = iter(iterable)

# infinite loop
while True:
    try:
        # get the next item
        element = next(iter_obj)
        # do something with element
    except StopIteration:
        # if StopIteration is raised, break from loop
        break

class PowTwo:
    """Class to implement an iterator
    of powers of two"""

    def __init__(self, max = 0):
        self.max = max

    def __iter__(self):
        self.n = 0
        return self

    def __next__(self):
        if self.n <= self.max:
            result = 2 ** self.n
            self.n += 1
            return result
        else:
            raise StopIteration

# create an iterator and iterate through it as follows.

>>> a = PowTwo(4)
>>> i = iter(a)
>>> next(i)
1
>>> next(i)
2
>>> next(i)
4
>>> next(i)
8
>>> next(i)
16
>>> next(i)
Traceback (most recent call last):
...
StopIteration

# use a for loop to iterate over our iterator class.

>>> for i in PowTwo(5):
...     print(i)

import itertools
infectious_big_list = [['A{}'.format(i) for i in range(10, 100)], ['A0{}'.format(i) for i in range(0, 10)], ['B{}'.format(i) for i in range(10, 100)], ['B0{}'.format(i) for i in range(0, 10)]]
infectious_disease_cd = list(itertools.chain.from_iterable(infectious_big_list))
mapping_illustrated_disease = {x:True for x in infectious_disease_cd}

def multiply_strings(x, col=('med_code', 'med_usage')):
    # print(type(x))
    ls_x = x[col[0]].split(',')
    ls_y = x[col[1]].split(',')
    # assert len(ls_x) == len(ls_y)
    if len(ls_x) != len(ls_y):
        return x[col[0]]
    ls_new = []
    for i in range(len(ls_x)):
        # remove stop words
        if int(ls_y[i]) < 50:
            ls_new.append((ls_x[i]+',')*int(ls_y[i]))
        else:
            ls_new.append((ls_x[i]+','))
    return ''.join(ls_new)
df.apply(lambda x:multiply_strings(x), axis=1)

import re
[m.start() for m in re.finditer('test', 'test test test test')]

greeting = ['hello','my','name','is','bob','how','are','you','my','name','is']

def find_sub_list(sl,l):
    sll=len(sl)
    for ind in (i for i,e in enumerate(l) if e==sl[0]):
        if l[ind:ind+sll]==sl:
            return ind,ind+sll-1

print find_sub_list(['my','name','is'], greeting)

a = [1,2,3]
b = [3,2,1]
a.sort()
b.sort()
a == b

testdata = [list(x) for x in set(tuple(x) for x in testdata)]

[x+y for x in [1,2,3] for y in [4,5,6]]
# equal to
res =[]
for x in [1,2,3]:
    for y in [4,5,6]:
        res.append(x+y)
[y+1 for x in [[1,2],[2,2],[3,2]] for y in x]

# equal to
res =[]
for x in [[1,2],[2,2],[3,2]]:
    for y in x:
        res.append(1+y)

list.remove('value')

import itertools
a = [["a","b"], ["c"]]
print list(itertools.chain.from_iterable(a))
# or
lambda: (lambda b: map(b.extend, big_list))([])

a = [1,2,3,2,1,5,6,5,5,5]
import collections
print [item for item, count in collections.Counter(a).items() if count > 1]

[a.lower() for a in x=['Hello', 'World', 18, 'Apple', None] if isinstance(a,str)]

with open(r'y:\codes\data\smart_beta_etf_list.txt', 'rb') as f:
etf_list = f.readlines()
etf_list = [x.strip() for x in etf_list]
# you may also want to remove whitespace characters like `\n` at the end of each line

thefile = open('test.txt', 'w')
import pickle

with open('outfile', 'wb') as fp:
    pickle.dump(itemlist, fp)
# To read it back:

with open ('outfile', 'rb') as fp:
    itemlist = pickle.load(fp)

values = [u'股市']
import codecs
with codecs.open("file.txt", "w", encoding="utf-8") as d:
    d.write(str(x0))

thefile.write("%s\n" % item)

text.rsplit(' ', 1)[0]

text.split(' ', 1)[0]
>>>a.split('.',1)
['alvy','test.txt']
后面多了一个参数1，以第一个'.'分界，分成两个字符串，组成一个list
>>>a.rsplit('.',1)
['alvy.test','txt']
现在是rsplit函数，从右边第一个'.'分界，分成两个字符串，组成一个list

string.split(",")

import re
re.split('; |, |\*|\n',str)

def odd():
    print('step 1')

    yield 1
    print('step 2')
    yield(3)
    print('step 3')
    yield(5)
>>> o = odd()
>>> next(o)
step 1
1
>>> next(o)
step 2
3
>>> next(o)
step 3
5
>>> next(o)
Traceback (most recent call last):

  File "<stdin>", line 1, in <module>
StopIteration

def a_function():
    "when called, returns generator object"
    yield

a_generator = (i for i in range(0))

b"abcde".decode("utf-8")

a=set([1,2,3])
element = a.pop(0)

list(a)[0]

from random import sample

def ForLoop(s):
    for e in s:
        break
    return e

def IterNext(s):
    return next(iter(s))

def ListIndex(s):
    return list(s)[0]

def PopAdd(s):
    e = s.pop()
    s.add(e)
    return e

def RandomSample(s):
    return sample(s, 1)

def SetUnpacking(s):
    e, *_ = s
    return e

from simple_benchmark import benchmark

b = benchmark([ForLoop, IterNext, ListIndex, PopAdd, RandomSample, SetUnpacking],
              {2**i: set(range(2**i)) for i in range(1, 20)},
              argument_name='set size',
              function_aliases={first: 'First'})

b.plot()

dict1.update(dict2)

with open('./data/disease.pkl', 'wb') as f:
    pickle.dump(dict_intravenous_thrombolysis, f)

import json
with open('multiple_paths.json', 'w', encoding='utf-8') as fp:
    js_obj = json.dumps(filtered_dict)
    fp.write(js_obj)

for name, age in word2id.items():    # for name, age in list.items():  (for Python 3.x)
    if age == 16116:
        print(name)

# or
mydict = {'george':16,'amber':19}
print(list(mydict.keys())[list(mydict.values()).index(16)]) # Prints george

print(list(word2id.keys())[list(word2id.values()).index(16116)]) # Prints

value = {}
for key in finance_vocab:
    value[key] = dict_vocab.get(key)

filtered_dict = {k:v for k,v in dict.items() if v<0}

visited = dict.fromkeys(self.graph, False)

'红鲱鱼招股书' in g.graph.values()

any('波动性' in v for v in g.graph.values())
# or
def in_values(s, d):
    """Does `s` appear in any of the values in `d`?"""
    for v in d.values():
        if s in v:
            return True
    return False

in_values('cow', animals)

s = [(k, d[k]) for k in sorted(d, key=d.get, reverse=True)]

d = defaultdict(list)
for name in g.graph.keys():
  key = len(g.graph[name])
  d[name] = key

from collections import defaultdict
d = defaultdict(list)
for k, v in list(graph.out_edges('财务管理')):
    d[k].append(v)
# or
d = {}
for k, v in list(graph.out_edges('财务管理')):
    d.setdefault(k,[]).append(v)

from copy import copy
my_dict = copy(another_dict)
my_dict.update((x, y*2) for x, y in my_dict.items())

In [231]: d
Out[231]:
defaultdict(list,
            {'上海证券交易所上市公司': 383,
             '各证券交易所上市公司': 37,
             '深圳证券交易所上市公司': 511,
             '证券': 64,
             '证券交易所': 8})

sum(d.values())

import json
# writing
json.dump(yourdict, open(filename, 'w'))
# reading
yourdict = json.load(open(filename))

print `a`
print repr(a)

import os
import spacy
print(os.path.dirname(spacy.__file__))

try:
    xx
except cx.DatabaseError:
    continue

response = None
error = None
while response is None:
  try:
    response = doing_something()
    if response is not None:
      if 'good' in response:
        print("successfully uploaded")
      else:
        exit("reason %s"%response)
  except HttpError as e:
    if e.code in RETRIABLE_STATUS_CODES:
      error = 'A retriable HTTP error %d occurred:\n%s' % (e.resp.status,
                                                             e.content)
    else:
      raise
  except RETRIABLE_EXCEPTIONS as e:
    error = 'A retriable error occurred: %s' % e

    if error is not None:
      print error
      retry += 1
      if retry > MAX_RETRIES:
        exit('No longer attempting to retry.')

      max_sleep = 2 ** retry
      sleep_seconds = random.random() * max_sleep
      print 'Sleeping %f seconds and then retrying...' % sleep_seconds
      time.sleep(sleep_seconds)

import urllib2

req = urllib2.Request('http://www.python.org/fish.html')
try:
    resp = urllib2.urlopen(req)
except urllib2.HTTPError as e:
    if e.code == 404:
        # do something...
    else:
        # ...
except urllib2.URLError as e:
    # Not an HTTP-specific error (e.g. connection refused)
    # ...
else:
    # 200
    body = resp.read()

class ConstraintError(Exception):
    def __init__(self, arg):
        self.args = arg


if error:
    raise ConstraintError("error")


class Networkerror(RuntimeError):
    def __init__(self, arg):
        self.args = arg


try:
    raise Networkerror("Bad hostname")
except Networkerror,e:
    print e.args
    print e.message

>>> def divide(x, y):
...     try:
...         result = x / y
...     except ZeroDivisionError:
...         print "division by zero!"
...     else:
...         print "result is", result
...     finally:
...         print "executing finally clause"
...
>>> divide(2, 1)
result is 2
executing finally clause
>>> divide(2, 0)
division by zero!
executing finally clause
>>> divide("2", "1")
executing finally clause
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "<stdin>", line 3, in divide
TypeError: unsupported operand type(s) for /: 'str' and 'str'

from googletrans import Translator
translator = Translator(proxies ={
             'http': 'http://192.168.1.126:1080',
             'https': 'http://192.168.1.126:1080'
         }
)
input_text_language_0 = translator.detect(input_text_0).lang

from difflib import SequenceMatcher
s_1 = 'Mohan Mehta'
s_2 = 'Mohan Mehte'
print(SequenceMatcher(a=s_1,b=s_2).ratio())
0.909090909091

if not text:
  print('text is empty')

from nltk.corpus import wordnet
if not wordnet.synsets(word) and not word.isdigit()

punct = set(u''':!),.:;?]}¢'"、。〉》」』】〕〗〞︰︱︳﹐､﹒
﹔﹕﹖﹗﹚﹜﹞！），．：；？｜｝︴︶︸︺︼︾﹀﹂﹄﹏､～￠
々‖•·ˇˉ―--′’”([{£¥'"‵〈《「『【〔〖（［｛￡￥〝︵︷︹︻
︽︿﹁﹃﹙﹛﹝（｛“‘-—_…''')

str_in = u"小明硕士毕业于中国科学院计算所，\
后在日本京都大学深造，凭借过人天赋，旁人若在另一方面爱他，他每即躲开。"

# 对str/unicode
filterpunt = lambda s: ''.join(filter(lambda x: x not in punct, s))
# 对list
filterpuntl = lambda l: list(filter(lambda x: x not in punct, l))
seg_list = jieba.cut(str_in, cut_all=False)
sent_list = filterpuntl(seg_list)

python -m jieba news.txt > cut_result.txt

from nltk import tokenize as n_tokenize
sent= n_tokenize.sent_tokenize(page)
# or
sent_list = page.split()

[x for x in t if x not in s if x.isdigit()]
l = [22, 13, 45, 50, 98, 69, 43, 44, 1]
[True if x >= 45 else False for x in l]

str.isdigit()

" ".join((str1, str2))

#!/usr/bin/python

str = "0000000this is string example....wow!!!0000000";
print(str.strip( '0' ))

nilbody

import linecache
linecache.getline('Sample.txt', Number_of_Line)

import datetime
print(
    datetime.datetime.fromtimestamp(
        int("1284101485")
    ).strftime('%Y-%m-%d %H:%M:%S')
)

>>> pd.to_datetime('13000101', format='%Y%m%d', errors='ignore')
datetime.datetime(1300, 1, 1, 0, 0)

tz = pytz.timezone('America/Los_Angeles')
#date = date.today()
now = datetime.now()
los_angeles_time = datetime.now(tz)

from tqdm import tqdm
from time import sleep
for i in tqdm(range(10)):
    sleep(0.1)

# enumerate
for i in enumerate(tqdm(list)):
    do things()
# pandas
df = pd.DataFrame(np.random.randint(0, int(1e8), (10000, 1000)))

# Create and register a new `tqdm` instance with `pandas`
# (can use tqdm_gui, optional kwargs, etc.)
tqdm.pandas()

# Now you can use `progress_apply` instead of `apply`
df.groupby(0).progress_apply(lambda x: x**2)

time.strptime(string[, format])

#Timestamp to string:i
str_timestamp = pd.to_datetime(Timestamp, format = '%Y%m%d')
str_timestamp = str_timestamp.strftime('%Y-%m-%d')

pd.DatetimeIndex.get_loc(datetime)

TimeStamp +/- pd.DateOffset(years=1)
pd.Timedelta(days=365) #allowed keywords are [weeks, days, hours, minutes, seconds, milliseconds, microseconds, nanoseconds]

start = ls_dates[d]
start = pd.to_datetime(start)
period = start + pd.DateOffset(30)
print(start, period)
df_range = df_simul[(df_simul.index < period) & (df_simul.index > start)]

from copy import copy

>>> import pprint
>>> tup = ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead',
... ('parrot', ('fresh fruit',))))))))
>>> stuff = ['a' * 10, tup, ['a' * 30, 'b' * 30], ['c' * 20, 'd' * 20]]
>>> pprint.pprint(stuff)
['aaaaaaaaaa',
 ('spam',
  ('eggs',
   ('lumberjack',
    ('knights', ('ni', ('dead', ('parrot', ('fresh fruit',)))))))),
 ['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'],
 ['cccccccccccccccccccc', 'dddddddddddddddddddd']]

0.00011001100110011001100110011001100110011001100110011010

>>> round(2.675, 2)
2.67

import numpy as np
np.set_printoptions(suppress=True)
%precision %.4g

if os.path.exists("demofile.txt"):
  os.remove("demofile.txt")

import os
def change_filename(dir_name, filename, suffix, extension=None):
    # name = filename.split('/')[-1]
    path, ext = os.path.splitext(filename)
    if extension is None:
        extension = ext
    return os.path.join(dir_name, path + suffix + extension)

import os
os.move(source_file_path, destination)

import sys, os
# run python file.py
ROOTDIR = os.path.join(os.path.dirname(__file__), os.pardir)
sys.path.append(os.path.join(ROOTDIR, "lib"))
# run in python
ROOTDIR = os.path.join(os.path.dirname("__file__"), os.pardir)

import os
import tempfile
TEMP_FOLDER = tempfile.gettempdir()
print('Folder "{}" will be used to save temporary dictionary and corpus.'.format(TEMP_FOLDER))

import os
from os.path import join, getsize
for root, dirs, files in os.walk('/home/weiwu/share/deep_learning/data/enwiki'):
    print(root, "consumes, ")
    print(sum([getsize(join(root, name)) for name in files]), '\s')
    print("bytes in", len(files), "non-directory files")

os.path.isfile(os.path.join(path,name))

import os
cwd = os.getcwd()

from tempfile import gettempdir
tmp_dir = gettempdir()

>>> f = open('workfile', 'w')
>>> print f
<open file 'workfile', mode 'w' at 80a0960>

f = open('filename.txt', 'a')
f.write("stuff")
f.close()

fp = open("file")
for i, line in enumerate(fp):
    if i == 25:
        # 26th line
    elif i == 29:
        # 30th line
    elif i > 29:
        break
fp.close()
# Note that i == n-1 for the nth line.

# In Python 2.6 or later:
with open("file") as fp:
    for i, line in enumerate(fp):
        if i == 25:
            # 26th line
        elif i == 29:
            # 30th line
        elif i > 29:
            break

pip install mysqlclient

pip search mysql-connector | grep --color mysql-connector-python
pip install mysql-connector-python-rf

>>>
>>> x = [1, 2, 3]
>>> y = [4, 5, 6]
>>> zipped = zip(x, y)
>>> zipped
[(1, 4), (2, 5), (3, 6)]
>>> x2, y2 = zip(*zipped)
>>> x == list(x2) and y == list(y2)
True

>>> x = [1, 2, 3]
>>> y = [4, 5, 6]
>>> zipped = zip(x, y)
>>> zipped
[(1, 4), (2, 5), (3, 6)]
In [172]: dict(zipped)
Out[179]: {1: 4, 2: 5, 3: 6}

>>> args = ("two", 3,5)
>>> test_args_kwargs(*args)
arg1: two
arg2: 3
arg3: 5

>>> kwargs = {"arg3": 3, "arg2": "two","arg1":5}
>>> test_args_kwargs(**kwargs)
arg1: 5
arg2: two
arg3: 3

class argparse(dict):
    """
    Example:
    m = Map({'first_name': 'Eduardo'}, last_name='Pool', age=24, sports=['Soccer'])
    """
    def __init__(self, *args, **kwargs):
        super(argparse, self).__init__(*args, **kwargs)
        for arg in args:
            if isinstance(arg, dict):
                for k, v in arg.iteritems():
                    self[k] = v

        if kwargs:
            for k, v in kwargs.iteritems():
                self[k] = v
    def add_argument(self, *args, **kwargs):
        # super(Map, self).__init__(*args, **kwargs)
        for i in args:
            self[i.strip('-')] = kwargs.get('default', None)
            if 'action' in kwargs:
                if kwargs['action'] == 'store_true':
                    self[i.strip('-')] = True
                else:
                    self[i.strip('-')] = False
    def parse_args(self):
        return self

    def __getattr__(self, attr):
        return self.get(attr)

    def __setattr__(self, key, value):
        self.__setitem__(key, value)

    def __setitem__(self, key, value):
        super(argparse, self).__setitem__(key, value)
        self.__dict__.update({key: value})

    def __delattr__(self, item):
        self.__delitem__(item)

    def __delitem__(self, key):
        super(argparse, self).__delitem__(key)
        del self.__dict__[key]

# config.yaml
engine:
  user:
    'jack'
  password:
    'password'

import yaml
with open(r'config.yaml', 'rb') as f:
    config = yaml.load(f)

from ylib.yaml_config import Configuraion
config = Configuraion()
config.load('../config.yaml')
print(config.__str__)

USER_AGENT = config.USER_AGENT
DOMAIN = config.DOMAIN
BLACK_DOMAIN = config.BLACK_DOMAIN
URL_SEARCH = config.URL_SEARCH

import logging
logger = logging.getLogger()
handler = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
handler.setFormatter(formatter)
if not logger.handler:
    logger.addHandler(handler)
logger.setLevel(logging.DEBUG)
logger
# or
logging.basicConfig(
    format='%(asctime)s : %(levelname)s : %(message)s', level=logging.DEBUG)

# at the end of the program
handler.close()
logger.removeHandler(handler)

from ylib import ylog
import logging

ylog.set_level(logging.DEBUG)
ylog.console_on()
ylog.filelog_on("app")

conda install pandas-datareader

import pandas as pd
import datetime as dt
import numpy as np
from pandas_datareader import data as web

data = pd.DataFrame()
symbols = ['GLD', 'GDX']
for sym in symbols:
    data[sym] = web.DataReader(sym, data_source='yahoo', start='20100510')['Adj Close']
data = data.dropna()

pip install snakeviz

$ python -m cProfile -o profile.stats test.py

# test.py
from random import randint
max_size = 10**4
data = [randint(0, max_size) for _ in range(max_size)]
test = lambda: insertion_sort(data)

$ snakeviz profile.stats

pip install opencv-python torch

pip install *.whl

pip install -U pip

[global]
#index-url=https://pypi.mirrors.ustc.edu.cn/simple/
#index-url=https://pypi.python.org/simple/
index-url=http://mirrors.aliyun.com/pypi/simple/
#index-url=https://pypi.gocept.com/pypi/simple/
#index-url=https://mirror.picosecond.org/pypi/simple/
[install]
trusted-host=mirrors.aliyun.com
#trusted-host=mirrors.ustc.edu.cn

pip freeze > requirements.txt

--index-url http://mirrors.aliyun.com/pypi/simple/
pandas
pylint
pep8
sphinx
ipython
numpy
ipdb
mock
nose

pyenv versions

import os

env_dist = os.environ # environ是在os.py中定义的一个dict environ = {}

print(env_dist.get('JAVA_HOME'))
print(env_dist['JAVA_HOME'])

import os
os.path.exists(test_file.txt)
os.path.isfile("test-data")
export_dir = "export/"
if not os.path.exists(export_dir):
    os.mkdir(export_dir)

# os.listdir() 方法用于返回指定的文件夹包含的文件或文件夹的名字的列表。这个列表以字母顺序。 它不包括 '.' 和'..' 即使它在文件夹中.
path = os.getcwd()
dirs = os.listdir(path)

import os
if os.access("/file/path/foo.txt", os.F_OK):
    print "Given file path is exist."

if os.access("/file/path/foo.txt", os.R_OK):
    print "File is accessible to read"

if os.access("/file/path/foo.txt", os.W_OK):
    print "File is accessible to write"

if os.access("/file/path/foo.txt", os.X_OK):
    print "File is accessible to execute"

#!/usr/bin/python3

import sys

print ('参数个数为:', len(sys.argv), '个参数。')
print ('参数列表:', str(sys.argv))

$ python3 test.py arg1 arg2 arg3
参数个数为: 4 个参数。
参数列表: ['test.py', 'arg1', 'arg2', 'arg3']

pd.options.mode.chained_assignment = None  # default='warn'

import warnings
warnings.filterwarnings("ignore", message="numpy.dtype size changed")
warnings.filterwarnings("ignore", message="numpy.ufunc size changed")

from __future__ import print_function
from __future__ import division
from __future__ import unicode_literals
from __future__ import absolute_import

import inspect
inspect.getfile(Module)

# For Python 2 use built-in function reload():

reload(module)
# For Python 2 and 3.2–3.3 use reload from module imp:

import imp
imp.reload(module)
# or
import importlib
importlib.reload(module)

# folder1
#    \__init__.py
#    \State.py
#    \StateMachine.py
#    \mouse_folder
#        \MouseAction.py
import os,sys,inspect
currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
parentdir = os.path.dirname(currentdir)
sys.path.insert(0,parentdir+'\\mouse')
sys.path.insert(0,parentdir)

from State import State
from StateMachine import StateMachine
from MouseAction import MouseAction

import os
print os.path.abspath(ylib.__file__)

mkvirtual -p python3 ENVNAME

conda uninstall tqdm
easy_install git+https://github.com/quantopian/zipline.git

vim ~/.condarc
channels:
  - https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/free/
  - https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/
  - https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/msys2/
show_channel_urls: true

# or
conda config --add channels 'https://mirrors.tuna.tsinghua.edu.cn/anaconda/pkgs/free/'
conda config --set show_channel_urls yes

import pkg_resources
pkg_resources.require("gensim")  # latest installed version
pkg_resources.require("gensim==3.7.2")  # this exact version
pkg_resources.require("gensim>=3.7.2")  # this version or higher

conda remove --name myenv --all
# You may instead use conda env remove --name myenv.
# To verify that the environment was removed, in your terminal window or an Anaconda Prompt, run:
conda info --envs

# 创建 conda 虚拟环境（ :code:`env_name` 是您希望创建的虚拟环境名）
$ conda create --name env_name python=3.5

# 如您想创建一个名为rqalpha的虚拟环境
$ conda create --name rqalpha python=3.5

# 使用 conda 虚拟环境
$ source activate env_name
# 如果是 Windows 环境下 直接执行 activcate
$ activate env_name

# 退出 conda 虚拟环境
$ source deactivate env_name
# 如果是 Windows 环境下 直接执行 deactivate
$ deactivate env_name

# 删除 conda 虚拟环境
$ conda-env remove --name env_name

# add conda for all users
sudo ln -s /share/anaconda3/etc/profile.d/conda.sh /etc/profile.d/conda.sh
# Previous to conda 4.4, the recommended way to activate conda was to modify PATH in
# your ~/.bashrc file.  You should manually remove the line that looks like

    export PATH="/share/anaconda3/bin:$PATH"

# ^^^ The above line should NO LONGER be in your ~/.bashrc file! ^^^

from future import division
print “%s %.4f%%” % (sid, (len(not_close)/len(ctp)))

import subprocess
try:
    subprocess.call(["java",
                     "-jar", grobid_jar,
                     # Avoid OutOfMemoryException
                     "-Xmx1024m",
                     "-gH", grobid_home,
                     "-gP", os.path.join(grobid_home,
                                         "config/grobid.properties"),
                     "-dIn", pdf_folder,
                     "-exe", "processReferences"])
    return True
except subprocess.CalledProcessError:
    return False

pd.set_option('display.float_format', lambda x: '%.4f' % x)

df.idxmax(axis=1)

pd.set_option('display.max_rows', 500)
pd.set_option('display.max_columns', 500)
pd.set_option('display.width', 1000)

>>> df = pd.DataFrame([[i] for i in range(10)], columns=['num'])
>>> df
    num
0    0
1    1
2    2
3    3

>>> def powers(x):
>>>     return x, x**2, x**3, x**4, x**5, x**6

>>> df['p1'], df['p2'], df['p3'], df['p4'], df['p5'], df['p6'] = \
>>>     zip(*df['num'].map(powers))

today = datetime.datetime.today()
the_last_day_of_previous_month = today.datetime.timedelta(days=1)
the_first_day_of_previous_month = the_last_day_of_previous_month.replace(day=1)

today - pd.Timedelta(1, unit='M')

df_patient[['pid','med_clinic_id']].groupby('pid').apply(lambda x:x['med_clinic_id'].tolist()).to_dict()

# select the first row in each group, group have multindex.
df_item.groupby(level=0, group_keys=False).apply(lambda x: x.sort_values('abs_diff', ascending=False)).groupby(level=0).head(3)
df_most_visited_hospitals.sort_values(['PERSON_ID','DISCHARGE_DATE'],ascending=True).groupby(['PERSON_ID', 'ICD3']).first()

df_patient = df_patient.groupby(['hos_id', 'disease']).filter(lambda x:x['person_id'].unique().size>=3)

for group in df2.groupby('type'):
    print(group_id, group[0])
    data = group[1]
    if group[0] == 'A':
        print group[1].min()

df = pd.DataFrame({'A':'foo foo foo bar bar bar'.split(),
                   'B':[0.1, 0.5, 1.0]*2})

df['C'] = df.groupby(['A'])['B'].transform(
                     lambda x: pd.qcut(x, 3, labels=range(1,4)))
print(df)
# if error ValueError: Length mismatch: Expected axis has 5564 elements, new values have 78421 elements
# Groupby does not group the NaNs:
df_patient['disease_code'].fillna('unk', inplace=True)

df_patient.reset_index().groupby('disease_code').apply(lambda x:['med_clinic_id'].count()/x['person_id'].nunique())

pd.qcut(df['column'].rank(method='first'), nbins)

df = pd.DataFrame([[1,2,3],[4,5,6]], columns=['a','b','c'])
def rowFunc(row):
    return row['a'] + row['b'] * row['c']

def rowIndex(row):
    return row.name
df['d'] = df.apply(rowFunc, axis=1)
df['rowIndex'] = df.apply(rowIndex, axis=1)
df
Out[182]:
   a  b  c   d  rowIndex
0  1  2  3   7         0
1  4  5  6  34         1

person_id = df_disease_sample['person_id'].unique()
from itertools import permutations
perm = permutations(person_id, 2)
df_person_similarity_adj_matrix = pd.DataFrame(index=perm, columns=['similarity'])
df_person_similarity_adj_matrix

index.drop('value')

df_suspicious_pairs[df_suspicious_pairs[['person_id_1', 'person_id_2']].isin(['11076976']).any(axis=1)]
def add_edges(G, row, df_suspicious_pairs):
    person_id = row.index
    df_targets = df_suspicious_pairs[df_suspicious_pairs[['person_id_1', 'person_id_2']].isin(person_id).any(axis=1)]
    G.add_edges_from([tuple(x) for x in df_targets[['person_id_1', 'person_id_2']].values])


df_suspicious_person.progress_apply(lambda x: add_edges(G, x, df_suspicious_pairs))

df_disease_sample[~df_disease_sample['treatment_code'].str.contains('DE')]

med_hos_id_mapping.set_index('med_clinic_id')['hos_id'].to_dict()

pd.DataFrame.from_records([tuples])

similarity_rank.index = pd.MultiIndex.from_tuples(similarity_rank.index)

df = pd.DataFrame(dict(A=list('aabb'), B=[1, 2, 3, 4], C=[0, 9, 0, 9]))
# groupby is the standard use aggregater
df.groupby('A').mean()
# maybe you want these values broadcast across the whole group and return something with the same index as what you started with.
# use transform

df.groupby('A').transform('mean')
# is equalvilent to groupby("A").mean()
df.set_index('A').groupby(level='A').transform('mean')
# set column equal to groupby mean
df_item_dis_lv['mean'] = df_item_dis_lv.groupby(['name', 'dis', 'JGDJ'])['MODEL_JE'].transform('mean')
# agg is used when you have specific things you want to run for different columns or more than one thing run on the same column.

df.groupby('A').agg(['mean', 'std'])

df.groupby('a')['b'].apply(list)

import cx_Oracle as cx
import pandas as pd
import numpy as np
import os
from tqdm import *
import os
from sqlalchemy import create_engine


os.environ['NLS_LANG'] = 'SIMPLIFIED CHINESE_CHINA.UTF8'

engine = create_engine('oracle://MMAPV41:MMAPV411556@192.168.4.32:1521/orcl?charset=utf8')

conn=cx.connect('MMAPV41/MMAPV411556@192.168.4.32/orcl')
sql_regist = """
select med_clinic_id, person_id, person_nm, person_sex,
person_age, in_hosp_date, out_hosp_date,
med_ser_org_no, clinic_type, in_diag_dis_nm, out_diag_doc_cd,
med_amout, hosp_lev from t_kc21
"""
df_regist = pd.read_sql_query(sql_regist, engine)

s_med_clinic_id = pd.read_pickle('med_clinic_id.pkl')
n = 100
sql_regist = """
select med_clinic_id, person_id, person_nm, person_sex,
person_age, in_hosp_date, out_hosp_date,
med_ser_org_no, clinic_type, in_diag_dis_nm, out_diag_doc_cd,
med_amout, hosp_lev from t_kc21 where med_clinic_id in (%s)
"""
df_regist = pd.DataFrame()
for i in tqdm(range(0, int(len(s_med_clinic_id)/100), n)):
    s = "'"+','.join(s_med_clinic_id.ix[i:(i+n)].values.flatten()).replace(',',"','")+"'"
    sql = sql_regist%(s)
    try:
        df_regist_iter = pd.read_sql(sql, conn)
        df_regist = df_regist.append(df_regist_iter)
    except UnicodeDecodeError:
        continue
df_regist.to_pickle("registration_data.pkl")

df_item = t_order.groupby(['name']).filter(lambda x:x['hos_id'].unique().size>=10)

df_simul_sample = df_simul_sample.resample('1H')['PERSON_ID'].agg(list)

df_patient = df_patient.rename(columns = lambda x: x.lower())

df_result = pd.DataFrame()
for idx, value in enumerate(groups):
    df_target_group = df_patient.loc[value]
    df_target_group.sort_values('入院日期', inplace=True)
    df_target_group['checkin_diff'] = df_target_group['入院日期'].diff()/np.timedelta64(1, 'D')
    df_target_group.reset_index(inplace=True)
    index = df_target_group[df_target_group['checkin_diff']<=3].index
    result = pd.concat([df_target_group.loc[index], df_target_group.loc[index-1]]).sort_values('入院日期')
    result.drop_duplicates(['个人ID','入院日期'], inplace=True)
    result['group'] = idx
    result['hospitals'] = result['机构'].unique().shape[0]
    df_result = df_result.append(result.drop('checkin_diff',axis=1))

writer = pd.ExcelWriter('nanjing_result.xlsx')
df_nanjing.to_excel(writer,'全部分组')
big_groups.to_excel(writer,'超大组')
writer.save()

diff_checkout_next_checkin['date'] = dataframe.loc[index, columns].stack().sort_values()
diff_checkout_next_checkin['diff'] = diff_checkout_next_checkin['date'].diff()/np.timedelta64(1, 'M')

df_nanjing.sort_values(['group_sn', '个人ID', '入院日期'], inplace=True)

df_nanjing = pd.read_excel('result.xlsx',dtype={'证件号':str,
                                                '个人ID':str},
                         parse_dates=['入院日期','出院日期'])

df_items_sum = df_items.groupby(['disease_code', 'soc_srt_dire_nm']).agg({'amount': 'sum'})
# Change: groupby state_office and divide by sum
df_items_ratio = df_items_sum.groupby(level=0).apply(lambda x:
                                                 100 * x / float(x.sum()))

group_size = df_nanjing.groupby(['group_sn'])['个人ID'].unique().apply(len)
big_groups = df_nanjing[df_nanjing['group_sn'].isin(group_size[group_size >= 8].index)]
small_groups = df_nanjing[df_nanjing['group_sn'].isin(group_size[group_size < 8].index)]

df = pd.DataFrame.from_dict({}, orient='index')

df_person_count = df_simul['PERSON_ID'].value_counts()
# filter person id count is less than 3 times
df_simul = df_simul[df_simul['PERSON_ID'].isin(df_person_count[df_person_count>3].index)]

df_sample1 = df_sample.sort_values(['person_id','discharge_date'],ascending=True).groupby(['person_id', 'icd3']).first().agg({'hosp_lev':pd.Series.value_counts})
df_sample1.groupby(['person_id']).sum().idxmax(axis=1)

s_count = df_patient.groupby('disease_code')['mon'].value_counts()
s_count.name = 'cnt_mon'
df_disease['cnt_mon'] = s_count.reset_index().pivot(index='disease_code', columns='mon', values='cnt_mon').idxmax(axis=1)

import pandas as pd
df = pd.DataFrame({'date': ['2013-04-01','2013-04-01','2013-04-01','2013-04-02', '2013-04-02'],
    'user_id': ['0001', '0001', '0002', '0002', '0002'],
    'duration': [30, 15, 20, 15, 30]})
df.groupby('date').agg({'user_id':pd.Series.nunique})

rels = ['疾病名称','诊疗大类2']
rels_cure = df_cure.groupby(rels).size().reset_index()[rels].values.tolist()

df_tree.fillna(method='pad', axis=1, inplace=True)

del df['column']