tcpdump -i eth1 host 192.168.1.1

tcpdump -i eth1 port 25

tcpdump -i eth1 net 192.168

tshark -i eth0

tshark tcp dst port 80

python -V
Python 2.7.12

sudo pip install virtualenv
source ./bin/active # 开启
# 此后再使用 pip install 时会安装在独立的目录下

sudo apt-get install libxml2-dev libxslt1-dev python-dev
pip install scrapy

scrapy startproject demo

$ tree demo
demo
├── demo
│   ├── __init__.py
│   ├── items.py
│   ├── middlewares.py
│   ├── pipelines.py
│   ├── settings.py
│   └── spiders
│       └── __init__.py
└── scrapy.cfg

2 directories, 7 files

cd demo
scrapy genspider example example.com   # 使用该命令安装模板生成 Spider

import scrapy
from scrapy.pipelines.images import ImagesPipeline

class ImagePipeline(ImagesPipeline):
    default_headers = {
        'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
    }

    # 对各个图片 URL 返回一个 Request
    def get_media_requests(self, item, info):
        try:
            for image_url in item['image_urls']:
                f = image_url.split('.')[-1]
                yield scrapy.Request(image_url, meta={'image_name': item['image_name'], 'format': f}, headers=self.default_headers)
        except Exception as error:
            print error

    # 当一个单独项目中的所有图片请求完成时 (success, image_info_or_failure)
    def item_completed(self, results, item, info):
        image_paths = [x['path'] for ok, x in results if ok]
        if not image_paths:
            # raise DropItem("Item contains no images")
            print "Image path no exist"
        return item


    # Override the convert_image method to disable image conversion

    # scrapy convert image to jpg 重写此方法，可以下载自定的图片格式，不过可能需要特殊处理格式
    # def convert_image(self, image, size=None):
    #     buf = StringIO()
    #     try:
    #         image.save(buf, image.format)
    #     except Exception, ex:
    #         raise ImageException("Cannot process image. Error: %s" % ex)
    #
    #     return image, buf

    # 默认情况下，使用 ImagePipeline 组件下载图片的时候，图片名称是以图片 URL 的 SHA1 值进行保存的。
    # scrapy 0.12 可以覆盖 image_key 方法，在此后版本中 使用 file_path 来自定义下载图片名称
    # def image_key(self, url):
    #     image_guid = hashlib.sha1(url).hexdigest()
    #     return 'full/%s.jpg' % (image_guid)

    # http://stackoverflow.com/questions/6194041/scrapy-image-download-how-to-use-custom-filename/22263951#22263951
    def file_path(self, request, response=None, info=None):
        name = request.meta['image_name']
        f = request.meta['format']
        return 'full/%s.jpg' % name

import random

from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware

from scrapy.conf import settings


class RandomUserAgentMiddleware(UserAgentMiddleware):

    def __init__(self, user_agent=''):
        self.user_agent = user_agent

    # 每一请求都会走这个函数，在这里随机挑选 UA
    def process_request(self, request, spider):
        ua = random.choice(settings.get('USER_AGENT_LIST'))
        if ua:
            print "******Current UserAgent: %s **************" % ua

            request.headers.setdefault("User-Agent", ua)


class ProxyMiddleware(object):
    def process_request(self, request, spider):
        request.meta['proxy'] = random.choice(settings.get('HTTP_PROXY_LIST'))

ITEM_PIPELINES = {
    'imdb.pipelines.MoviePipeline': 300,
    'imdb.image_pipeline.ImagePipeline': 300
}

export PATH="$HOME/.pyenv/bin:$PATH"
eval "$(pyenv init -)"
eval "$(pyenv virtualenv-init -)"

curl -L https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer | bash

cd ~
git clone git://github.com/yyuu/pyenv.git .pyenv
echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bashrc
echo 'export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc
echo 'eval "$(pyenv init -)"' >> ~/.bashrc
source ~/.bashrc

pyenv versions

pyenv install -l

$ pyenv install 2.7.3   # 安装 python
$ pyenv uninstall 2.7.3 # 卸载 python

$ pyenv global 2.7.3  # 设置全局的 Python 版本，通过将版本号写入 ~/.pyenv/version 文件的方式。
$ pyenv local 2.7.3 # 设置 Python 本地版本，通过将版本号写入当前目录下的 .python-version 文件的方式。通过这种方式设置的 Python 版本优先级较 global 高。

$ pyenv shell 2.7.3 # 设置面向 shell 的 Python 版本，通过设置当前 shell 的 PYENV_VERSION 环境变量的方式。这个版本的优先级比 local 和 global 都要高。–unset 参数可以用于取消当前 shell 设定的版本。
$ pyenv shell --unset

$ pyenv rehash  # 创建垫片路径（为所有已安装的可执行文件创建 shims，如：~/.pyenv/versions/*/bin/*，因此，每当你增删了 Python 版本或带有可执行文件的包（如 pip）以后，都应该执行一次本命令）

einverne@ev  ~  cd ~/.pyenv/plugins
einverne@ev  ~/.pyenv/plugins   master  ll
total 24K
drwxr-xr-x 4 einverne einverne 4.0K Apr 22 10:55 pyenv-doctor
drwxr-xr-x 5 einverne einverne 4.0K Apr 22 10:55 pyenv-installer
drwxr-xr-x 4 einverne einverne 4.0K Apr 22 10:55 pyenv-update
drwxr-xr-x 7 einverne einverne 4.0K Apr 22 10:55 pyenv-virtualenv
drwxr-xr-x 4 einverne einverne 4.0K Apr 22 10:55 pyenv-which-ext
drwxr-xr-x 5 einverne einverne 4.0K Apr 22 10:54 python-build

$ pyenv virtualenv 2.7.10 env-2.7.10

pyenv virtualenvs
pyenv activate env-name  # 激活虚拟环境
pyenv deactivate #退出虚拟环境，回到系统环境

pyenv uninstall my-virtual-env
rm -rf ~/.pyenv/versions/env-name  # 或者删除其真实目录

$ pyenv commands
activate
commands
completions
deactivate
doctor
exec
global
help
hooks
init
install
installer
local
offline-installer
prefix
rehash
root
shell
shims
uninstall
update                 # 更新 pyenv 及插件
version
--version
version-file
version-file-read
version-file-write
version-name
version-origin
versions
virtualenv
virtualenv-delete
virtualenv-init
virtualenv-prefix
virtualenvs
whence
which

[global]
index-url = http://mirrors.aliyun.com/pypi/simple/

[install]
trusted-host=mirrors.aliyun.com

KEYS pattern

EXISTS key

TYPE key

SELECT 1

MOVE key <db index>

RANDOMKEY

RENAME key new_key

SET key value
GET key

SETNX key value  # 如果 key 存在，则返回 0，如果设置成功返回 1

SETEX key seconds value

INCR key

INCRBY key increment

DECR key

DECRBY key decrement

INCRBYFLOAT key increment

APPEND key value

STRLEN key

MGET key [key ...]

MSET key value [key value ...]

SETRANGE key offset value  # 用 value 值覆盖给定 key 从 offset 开始所存储的字符串
MSETNX key value key value # 一次性设置多个值， SETNX 的 multi 版本
GETSET key 				# 设置 key 的值，并返回 key 的旧值
GETRANGE key start end  # 截取 start 和 end 偏移量的字符串

HSET key field value
HGET key field

HMSET key field value [field value ...]
HMGET key field [field ...]

HSETNX key field value

HGETALL key

HEXISTS key field

HINCRBY key field increment

HDEL key field [field ...]

HKEYS key

HVALS key

LPUSH key value [value ...]
RPUSH key value [value... ]

LPOP key
RPOP key

LLEN key

LRANGE key start stop

LREM key count value

LINDEX key index

LTRIM key start end

LINSERT key BEFORE | AFTER pivot value

RPOPLPUSH source destination

SADD key member [member ...]

SREM key member [member ...]

SPOP key

SRANDMEMBER key [count]

SMEMBERS key

SISMEMBER key member

SDIFF key [key ...]

SINTER key [key ..]

SUNION key [key...]

SCARD key

SDIFFSTORE destination key [key ...]
SINTERSTORE destination key [key ...]
SUNIONSTORE destination key [key...]

ZADD key score member [score member ...]

ZSCORE key member

ZRANGE key start stop [WITHSCORES]

ZREVRANGE key start stop

ZRANGEBYSCORE key min max

ZINCRBY key increment member

ZCARD key

ZCOUNT key min max

ZREM key member [member ...]

ZREMRANGEBYRANK key start stop

ZREMRANGEBYSCORE key min max

ZRANK key member

ZREVRANK key member

ZINTERSTORE destination numkeys key [key ...] [WEIGHTS weight [weight ...]] [AGGREGATE SUM | MIN | MAX]

MULTI
SADD ”user:1:following" 2
SADD "user:2:followers" 1
EXEC

EXPIRE key seconds

TTL key

PERSIST key

https://redis.io/clients

sudo apt-get install redis-server

sudo service redis-server status # 查看当前状态
sudo service redis-server stop/start # 等等来控制 redis-server 的状态

curl -O http://download.redis.io/redis-stable.tar.gz

tar xzvf redis-stable.tar.gz

cd redis-stable

make

make test

sudo make install

taskset -c 1 sudo make test

sudo mkdir /etc/redis

sudo cp redis.conf /etc/redis

sudo vim /etc/redis/redis.conf

# If you run Redis from upstart or systemd, Redis can interact with your
# supervision tree. Options:
#   supervised no      - no supervision interaction
#   supervised upstart - signal upstart by putting Redis into SIGSTOP mode
#   supervised systemd - signal systemd by writing READY=1 to $NOTIFY_SOCKET
#   supervised auto    - detect upstart or systemd method based on
#                        UPSTART_JOB or NOTIFY_SOCKET environment variables
# Note: these supervision methods only signal "process is ready."
#       They do not enable continuous liveness pings back to your supervisor.
supervised systemd

# The working directory.
#
# The DB will be written inside this directory, with the filename specified
# above using the 'dbfilename' configuration directive.
#
# The Append Only File will also be created inside this directory.
#
# Note that you must specify a directory here, not a file name.
dir /var/lib/redis

sudo vim /etc/systemd/system/redis.service

[Unit]
Description=Redis In-Memory Data Store
After=network.target

[Service]
User=redis
Group=redis
ExecStart=/usr/local/bin/redis-server /etc/redis/redis.conf
ExecStop=/usr/local/bin/redis-cli shutdown
Restart=always

[Install]
WantedBy=multi-user.target

sudo adduser --system --group --no-create-home redis

sudo mkdir /var/lib/redis

sudo chown redis:redis /var/lib/redis

sudo chmod 770 /var/lib/redis

sudo systemctl start redis

sudo systemctl status redis

sudo service redis status
● redis.service - Redis In-Memory Data Store
   Loaded: loaded (/etc/systemd/system/redis.service; disabled; vendor preset: enabled)
   Active: active (running) since Sat 2017-04-22 18:59:56 CST; 2s ago
 Main PID: 28750 (redis-server)
   CGroup: /system.slice/redis.service
           └─28750 /usr/local/bin/redis-server 127.0.0.1:6379

Apr 22 18:59:56 ev redis-server[28750]:   `-._    `-._`-.__.-'_.-'    _.-'
Apr 22 18:59:56 ev redis-server[28750]:       `-._    `-.__.-'    _.-'
Apr 22 18:59:56 ev redis-server[28750]:           `-._        _.-'
Apr 22 18:59:56 ev redis-server[28750]:               `-.__.-'
Apr 22 18:59:56 ev redis-server[28750]: 28750:M 22 Apr 18:59:56.445 # WARNING: The TCP backlog setting of 511 cannot be enforced because /proc/sys/net/core/somaxconn is set to the lower valu
Apr 22 18:59:56 ev redis-server[28750]: 28750:M 22 Apr 18:59:56.445 # Server started, Redis version 3.2.8
Apr 22 18:59:56 ev redis-server[28750]: 28750:M 22 Apr 18:59:56.445 # WARNING overcommit_memory is set to 0! Background save may fail under low memory condition. To fix this issue add 'vm.ov
Apr 22 18:59:56 ev redis-server[28750]: 28750:M 22 Apr 18:59:56.445 # WARNING you have Transparent Huge Pages (THP) support enabled in your kernel. This will create latency and memory usage
Apr 22 18:59:56 ev redis-server[28750]: 28750:M 22 Apr 18:59:56.445 * DB loaded from disk: 0.000 seconds
Apr 22 18:59:56 ev redis-server[28750]: 28750:M 22 Apr 18:59:56.445 * The server is now ready to accept connections on port 6379

redis-cli

127.0.0.1:6379> ping
PONG
127.0.0.1:6379> set test "It's working"
OK
127.0.0.1:6379> get test
"It's working"
127.0.0.1:6379> exit

sudo systemctl restart redis.service

127.0.0.1:6379> get test
"It's working"

sudo systemctl enable redis

df [OPTIONS] [FILE]

-a      全部文件系统列表
-h      方便阅读方式显示
-H      等于“-h”，但是计算式，1K=1000，而不是 1K=1024
-i      显示 inode 信息
-k      区块为 1024 字节
-l      只显示本地文件系统
-m      区块为 1048576 字节
--no-sync 忽略 sync 命令
-P      输出格式为 POSIX
--sync  在取得磁盘信息前，先执行 sync 命令
-T      文件系统类型

Filesystem     1K-blocks      Used Available Use% Mounted on
udev             8126360         0   8126360   0% /dev
tmpfs            1629376     75432   1553944   5% /run
/dev/sdb1      240230912 185617700  42387064  82% /
tmpfs            8146864    546884   7599980   7% /dev/shm
tmpfs               5120         4      5116   1% /run/lock
tmpfs            8146864         0   8146864   0% /sys/fs/cgroup
/dev/loop1         83712     83712         0 100% /snap/core/4206
/dev/loop4        259584    259584         0 100% /snap/electronic-wechat/7
cgmfs                100         0       100   0% /run/cgmanager/fs
tmpfs            1629376        72   1629304   1% /run/user/1000
/dev/sda3      723180576     70584 686351464   1% /media/user/add8bd89-da2a-4573-ac6e-7ec44f8c5414
/dev/loop5         84096     84096         0 100% /snap/core/4327
/dev/loop3         95872     95872         0 100% /snap/slack/6
/dev/loop6         88704     88704         0 100% /snap/core/4407

Filesystem      Size  Used Avail Use% Mounted on
udev            7.8G     0  7.8G   0% /dev
tmpfs           1.6G   74M  1.5G   5% /run
/dev/sdb1       230G  178G   41G  82% /
tmpfs           7.8G  534M  7.3G   7% /dev/shm
tmpfs           5.0M  4.0K  5.0M   1% /run/lock
tmpfs           7.8G     0  7.8G   0% /sys/fs/cgroup
/dev/loop1       82M   82M     0 100% /snap/core/4206
/dev/loop4      254M  254M     0 100% /snap/electronic-wechat/7
cgmfs           100K     0  100K   0% /run/cgmanager/fs
tmpfs           1.6G   72K  1.6G   1% /run/user/1000
/dev/sda3       690G   69M  655G   1% /media/mi/add8bd89-da2a-4573-ac6e-7ec44f8c5414
/dev/loop5       83M   83M     0 100% /snap/core/4327
/dev/loop3       94M   94M     0 100% /snap/slack/6
/dev/loop6       87M   87M     0 100% /snap/core/4407

Filesystem     Type      Size  Used Avail Use% Mounted on
udev           devtmpfs  7.8G     0  7.8G   0% /dev
tmpfs          tmpfs     1.6G   74M  1.5G   5% /run
/dev/sdb1      ext4      230G  178G   41G  82% /

df -i

Filesystem       Inodes   IUsed    IFree IUse% Mounted on
udev            2031887     539  2031348    1% /dev
tmpfs           2036709     941  2035768    1% /run
/dev/sdb1      15269888 1896147 13373741   13% /
tmpfs           2036709     497  2036212    1% /dev/shm
tmpfs           2036709       4  2036705    1% /run/lock
tmpfs           2036709      18  2036691    1% /sys/fs/cgroup
/dev/loop0        28782   28782        0  100% /snap/electronic-wechat/7
cgmfs           2036709      14  2036695    1% /run/cgmanager/fs
tmpfs           2036709     132  2036577    1% /run/user/1000
/dev/loop8        12810   12810        0  100% /snap/core/6034
/dev/sda1      14082048 1673302 12408746   12% /media/mi/3d1b7e3e-c184-4664-9555-2b088997f2c8
/dev/sda3      45932544      11 45932533    1% /media/mi/8803a3c6-1561-4957-b9b3-e60d5688d1a6
/dev/sdc       12229708      20 12229688    1% /media/mi/data

Filesystem     Inodes IUsed IFree IUse% Mounted on
udev             2.0M   520  2.0M    1% /dev
tmpfs            2.0M   888  2.0M    1% /run
/dev/sdb1         15M  1.8M   13M   12% /

df -T

df -t ext4

$ pip install celery

$ pip install -U celery[redis]  # -U 的意思是把所有指定的包都升级到最新的版本

BROKER_URL = 'redis://localhost:6379/0'

# task.py
from celery import Celery

broker = 'redis://127.0.0.1:6379/0'

app = Celery('tasks', broker=broker)

@app.task()
def add(x, y):
   return x + y

$ celery worker -A task -l info

>>> from task import add
>>> add.delay(1, 2)
<AsyncResult: 42ade14e-c7ed-4b8d-894c-1ca1ec7ca192>

>>> result = add.delay(1, 2)
>>> result.get()
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/usr/local/lib/python2.7/dist-packages/celery/result.py", line 169, in get
    no_ack=no_ack,
  File "/usr/local/lib/python2.7/dist-packages/celery/backends/base.py", line 604, in _is_disabled
    'No result backend configured.  '
NotImplementedError: No result backend configured.  Please see the documentation for more information.

# task.py
...
app = Celery('tasks', backend='redis://localhost', broker='redis://localhost//')
...

>>> from task import add
>>> result = add.delay(1,2)
>>> result.get()
3

pip install flower

easy_install flower

flower -A proj --port=5555

from datetime import timedelta
from celery.schedules import crontab

app.conf.beat_schedule = {
    # Executes every Monday morning at 7:30 A.M
    'add-every-monday-morning': {
        'task': 'tasks.add',
        'schedule': crontab(hour=7, minute=30, day_of_week=1),
        'args': (20, 20),
    },
    # execute every ten minutes
   'every_ten_minutes': {
        'task': 'worker.cntv.cntv_test',
        'schedule': timedelta(minutes=10),
        'args': ('args1',),
        'options': {
            'queue': 'queue_name'
        }
    },
}

celery beat -A celery_app.celery_config -s /path/to/celerybeat-schedule -l info

celery -A proj -Q queue_name purge      # 取消队列 queue_name 中的任务
# or
from proj.celery import app
app.control.purge()

If you really want to continue then you have to set the C_FORCE_ROOT
environment variable (but please think about this before you do).

User information: uid=0 euid=0 gid=0 egid=0

from celery import Celery, platforms
platforms.C_FORCE_ROOT = True

environment=C_FORCE_ROOT="true"

sudo lsof -i tcp:80
COMMAND     PID USER   FD   TYPE   DEVICE SIZE/OFF NODE NAME
docker-pr 14863 root    4u  IPv6 38693061      0t0  TCP *:http (LISTEN)

sudo lsof -p 14863
COMMAND     PID USER   FD      TYPE   DEVICE SIZE/OFF       NODE NAME
docker-pr 14863 root  cwd       DIR      8,0     4096          2 /
docker-pr 14863 root  rtd       DIR      8,0     4096          2 /
docker-pr 14863 root  txt       REG      8,0  3329080      17531 /usr/bin/docker-proxy
docker-pr 14863 root  mem       REG      8,0  1868984      20743 /lib/x86_64-linux-gnu/libc-2.23.so
docker-pr 14863 root  mem       REG      8,0   138696      11625 /lib/x86_64-linux-gnu/libpthread-2.23.so
docker-pr 14863 root  mem       REG      8,0   162632      10738 /lib/x86_64-linux-gnu/ld-2.23.so
docker-pr 14863 root    0r      CHR      1,3      0t0       8085 /dev/null
docker-pr 14863 root    1w      CHR      1,3      0t0       8085 /dev/null
docker-pr 14863 root    2w      CHR      1,3      0t0       8085 /dev/null
docker-pr 14863 root    4u     IPv6 38693061      0t0        TCP *:http (LISTEN)
docker-pr 14863 root    5u  a_inode     0,12        0       8082 [eventpoll]
docker-pr 14863 root   12r      REG      0,3        0 4026531993 net

lsof  [ -?abChKlnNOPRtUvVX ] [ -A A ] [ -c c ] [ +c c ] [ +|-d d ] [ +|-D D ] [ +|-e s ] [ +|-E ] [ +|-f [cfgGn] ] [ -F [f] ] [ -g [s] ] [ -i [i] ] [ -k k ] [ +|-L [l] ] [ +|-m m
   ] [ +|-M ] [ -o [o] ] [ -p s ] [ +|-r [t[m<fmt>]] ] [ -s [p:s] ] [ -S [t] ] [ -T [t] ] [ -u s ] [ +|-w ] [ -x [fl] ] [ -z [z] ] [ -Z [Z] ] [ -- ] [names]

默认 : 没有选项，lsof 列出活跃进程的所有打开文件
-i : 列出网络连接

sudo lsof -i
COMMAND     PID     USER   FD   TYPE   DEVICE SIZE/OFF NODE NAME
sshd       2972     root    3u  IPv4 18883553      0t0  TCP *:22 (LISTEN)
sshd       2972     root    4u  IPv6 18883562      0t0  TCP *:22 (LISTEN)
docker-pr 14852     root    4u  IPv6 38693034      0t0  TCP *:https (LISTEN)
docker-pr 14863     root    4u  IPv6 38693061      0t0  TCP *:http (LISTEN)

lsof -p 10075

lsof -u einverne

lsof -X | grep fastjson