add uploader

uploader/.gitignore

@@ -0,0 +1,62 @@
+
+# Created by https://www.gitignore.io/api/pycharm
+
+### PyCharm ###
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+.idea/
+# User-specific stuff:
+.idea/workspace.xml
+.idea/tasks.xml
+.idea/dictionaries
+.idea/vcs.xml
+.idea/jsLibraryMappings.xml
+
+# Sensitive or high-churn files:
+.idea/dataSources.ids
+.idea/dataSources.xml
+.idea/dataSources.local.xml
+.idea/sqlDataSources.xml
+.idea/dynamic.xml
+.idea/uiDesigner.xml
+
+# Gradle:
+.idea/gradle.xml
+.idea/libraries
+
+# Mongo Explorer plugin:
+.idea/mongoSettings.xml
+
+## File-based project format:
+*.iws
+
+## Plugin-specific files:
+
+# IntelliJ
+/out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+### PyCharm Patch ###
+# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
+
+# *.iml
+# modules.xml
+__pycache__
+
+*.db
+*.json
+config.py
+cache/
+venv/

uploader/README.md

@@ -0,0 +1,23 @@
+# Uploader
+
+上传文件到七牛和腾讯云存储
+
+## 配置
+
+参考 `config.py.example` 文件，修改 `API key`，修改 `targets` 为要上传的文件或目录
+
+## 运行
+
+```shell
+virtualenv -p python3 venv
+source venv/bin/activate
+pip install -r requirements.txt -I
+python upload.py
+```
+
+**crontab**
+
+```shell
+10 * * * * /path/to/uploader/cron.sh >> /var/log/uploader.log 2>&1
+```
+

uploader/config.py.example

@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+
+qn_access_key = "ACCESS_KEY"
+qn_secret_key = "SECRET_KEY"
+qn_bucket_name = "BUCKET_NAME"
+
+cos_app_id = ""
+cos_bucket_name = ""
+cos_secret_id = ""
+cos_key = ""
+
+targets = ['/var/www/test.json', '/var/www/test_dir']

uploader/cron.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+echo $(date)
+
+PWD="$(dirname $0)"
+
+echo "$PWD"
+
+cd "$PWD" || exit 1
+
+venv/bin/python upload.py

uploader/requirements.txt

@@ -0,0 +1,2 @@
+qiniu == 7.0.7
+requests == 2.9.1

uploader/upload.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+import os
+
+import config
+import uploader
+
+
+def upload(targets):
+    for target in targets:
+
+        if os.path.isdir(target):
+            upload([os.path.join(target, f) for f in os.listdir(target)])
+        else:
+            if os.path.exists(target):
+                uploader.upload(target)
+
+upload(config.targets)

uploader/uploader.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+import os
+
+import qiniu
+
+import config
+
+access_key = config.qn_access_key
+secret_key = config.qn_secret_key
+bucket_name = config.qn_bucket_name
+
+cos_app_id = config.cos_app_id
+cos_bucket_name = config.cos_bucket_name
+cos_secret_id = config.cos_secret_id
+cos_key = config.cos_key
+
+
+def upload(name):
+    upload_file(name)
+    upload_cos(name)
+
+
+# upload to qiniu
+def upload_file(file_name):
+    q = qiniu.Auth(access_key, secret_key)
+
+    key = os.path.basename(file_name)
+
+    token = q.upload_token(bucket_name, key)
+    ret, info = qiniu.put_file(token, key, file_name)
+    if ret is not None:
+        print(file_name + ' uploaded.')
+    else:
+        print(info)
+
+
+# upload to q-cloud cos
+def upload_cos(file):
+    headers = {
+        'Authorization': sign()
+    }
+    url = 'https://web.file.myqcloud.com/files/v1/' + cos_app_id + '/' + cos_bucket_name + '/' + os.path.basename(file)
+    data = {'op': 'upload', 'insertOnly': '0'}
+    files = {'filecontent': open(file, 'rb')}
+    import requests
+    r = requests.post(url, data=data, files=files, headers=headers)
+    print(r.text)
+
+
+def sign():
+    import hmac
+    import hashlib
+
+    # a=[appid]&b=[bucket]&k=[SecretID]&e=[expiredTime]&t=[currentTime]&r=[rand]&f=
+    import time
+    current_time = int(time.time())
+    sign_text = 'a=' + cos_app_id + '&b=' + cos_bucket_name + '&k=' + cos_secret_id + '&e=' + str(
+        current_time + 3600) + '&t=' + str(current_time) + '&r=123&f='
+    sign_tmp = hmac.new(cos_key.encode(), sign_text.encode(), hashlib.sha1).digest() + sign_text.encode()
+    import base64
+
+    return base64.b64encode(sign_tmp).decode()
+