初始提交: 通用数据同步服务
- 基于OpenResty的RESTful API服务 - 支持Cookie/Token等数据类型的存储和管理 - 登录认证保护 - Web管理界面 - 数据过期管理
This commit is contained in:
@@ -0,0 +1,165 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
过期检测脚本 - 定时检查数据过期状态并发送微信通知
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
|
||||
DATA_DIR = Path('/home/hermes/projects/data-sync-service/data')
|
||||
INDEX_FILE = DATA_DIR / 'index.json'
|
||||
|
||||
|
||||
def load_index():
|
||||
"""读取索引文件"""
|
||||
if not INDEX_FILE.exists():
|
||||
return {"tags": {}}
|
||||
|
||||
with open(INDEX_FILE, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def load_data_file(filename):
|
||||
"""读取数据文件"""
|
||||
filepath = DATA_DIR / filename
|
||||
if not filepath.exists():
|
||||
return None
|
||||
|
||||
with open(filepath, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def check_expiry():
|
||||
"""检查所有数据的过期状态"""
|
||||
index = load_index()
|
||||
|
||||
tags = index.get('tags', {})
|
||||
if not tags:
|
||||
print("📊 暂无数据")
|
||||
return
|
||||
|
||||
expired_list = []
|
||||
warning_list = []
|
||||
valid_list = []
|
||||
|
||||
for tag, info in tags.items():
|
||||
expires_at_str = info.get('expires_at', '')
|
||||
if not expires_at_str:
|
||||
continue
|
||||
|
||||
try:
|
||||
expires_at = datetime.strptime(expires_at_str, '%Y-%m-%d %H:%M:%S')
|
||||
now = datetime.now()
|
||||
remaining = (expires_at - now).total_seconds()
|
||||
remaining_hours = remaining / 3600
|
||||
remaining_days = remaining_hours / 24
|
||||
|
||||
# 读取完整数据(获取 login_url)
|
||||
file_data = load_data_file(info['file'])
|
||||
login_url = ''
|
||||
if file_data and 'metadata' in file_data:
|
||||
login_url = file_data['metadata'].get('login_url', '')
|
||||
|
||||
item = {
|
||||
'tag': tag,
|
||||
'type': info.get('type', 'unknown'),
|
||||
'expires_at': expires_at_str,
|
||||
'remaining_days': remaining_days,
|
||||
'login_url': login_url
|
||||
}
|
||||
|
||||
if remaining < 0:
|
||||
expired_list.append(item)
|
||||
elif remaining < 2 * 24 * 3600: # 少于2天
|
||||
warning_list.append(item)
|
||||
else:
|
||||
valid_list.append(item)
|
||||
|
||||
except Exception as e:
|
||||
print(f"⚠️ 解析时间失败: {tag} - {e}")
|
||||
|
||||
# 发送通知
|
||||
has_notification = False
|
||||
|
||||
# 1. 紧急通知(已过期)
|
||||
for item in expired_list:
|
||||
has_notification = True
|
||||
send_expired_notification(item)
|
||||
print("")
|
||||
|
||||
# 2. 提醒通知(即将过期)
|
||||
for item in warning_list:
|
||||
has_notification = True
|
||||
send_warning_notification(item)
|
||||
print("")
|
||||
|
||||
# 3. 汇总统计
|
||||
if not has_notification:
|
||||
print("✅ 所有数据状态正常")
|
||||
print(f"\n有效数据:{len(valid_list)} 个")
|
||||
for item in valid_list:
|
||||
print(f" - {item['tag']} (剩余 {item['remaining_days']:.1f} 天)")
|
||||
else:
|
||||
print(f"📊 数据状态统计")
|
||||
print(f" ❌ 已过期: {len(expired_list)} 个")
|
||||
print(f" ⚠️ 即将过期: {len(warning_list)} 个")
|
||||
print(f" ✅ 正常: {len(valid_list)} 个")
|
||||
|
||||
|
||||
def send_expired_notification(item):
|
||||
"""发送过期通知"""
|
||||
tag = item['tag']
|
||||
data_type = item['type']
|
||||
expires_at = item['expires_at']
|
||||
login_url = item['login_url']
|
||||
|
||||
# 生成配置 JSON
|
||||
config = {
|
||||
"serverUrl": "http://47.122.126.244:5001",
|
||||
"password": "admin123123123",
|
||||
"tag": tag,
|
||||
"type": data_type
|
||||
}
|
||||
config_json = json.dumps(config, indent=2, ensure_ascii=False)
|
||||
|
||||
message = f"""❌ 数据已过期
|
||||
|
||||
Tag: {tag}
|
||||
类型: {data_type}
|
||||
过期时间: {expires_at}
|
||||
|
||||
请重新登录同步:
|
||||
{login_url if login_url else '(无登录链接)'}
|
||||
|
||||
配置信息(点击复制):
|
||||
```json
|
||||
{config_json}
|
||||
```"""
|
||||
|
||||
print(message)
|
||||
|
||||
|
||||
def send_warning_notification(item):
|
||||
"""发送提醒通知"""
|
||||
tag = item['tag']
|
||||
data_type = item['type']
|
||||
expires_at = item['expires_at']
|
||||
remaining_days = item['remaining_days']
|
||||
login_url = item['login_url']
|
||||
|
||||
message = f"""⚠️ 数据即将过期
|
||||
|
||||
Tag: {tag}
|
||||
类型: {data_type}
|
||||
剩余时间: {remaining_days:.1f} 天
|
||||
过期时间: {expires_at}
|
||||
|
||||
建议重新登录同步:
|
||||
{login_url if login_url else '(无登录链接)'}"""
|
||||
|
||||
print(message)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
check_expiry()
|
||||
Executable
+145
@@ -0,0 +1,145 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
通用数据同步服务 - 过期检测脚本
|
||||
每天检查所有数据的过期状态,发送微信通知
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
DATA_DIR = Path("/home/hermes/projects/data-sync-service/data")
|
||||
SERVER_URL = "http://47.122.126.244:5001"
|
||||
|
||||
|
||||
def parse_datetime(dt_str):
|
||||
"""解析时间字符串"""
|
||||
try:
|
||||
return datetime.strptime(dt_str, "%Y-%m-%d %H:%M:%S")
|
||||
except:
|
||||
return None
|
||||
|
||||
|
||||
def load_index():
|
||||
"""加载索引文件"""
|
||||
index_file = DATA_DIR / "index.json"
|
||||
if not index_file.exists():
|
||||
return {"tags": []}
|
||||
|
||||
with open(index_file, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def load_data(filename):
|
||||
"""加载数据文件"""
|
||||
filepath = DATA_DIR / filename
|
||||
if not filepath.exists():
|
||||
return None
|
||||
|
||||
with open(filepath, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def format_config_block(tag, data_type):
|
||||
"""格式化配置代码块"""
|
||||
config = {
|
||||
"serverUrl": SERVER_URL,
|
||||
"password": "admin123123123",
|
||||
"tag": tag,
|
||||
"type": data_type
|
||||
}
|
||||
return json.dumps(config, indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
def send_expired_notification(tag, data_type, expires_at, login_url):
|
||||
"""发送过期通知"""
|
||||
config_json = format_config_block(tag, data_type)
|
||||
|
||||
message = f"""❌ 数据已过期
|
||||
|
||||
Tag: {tag}
|
||||
类型: {data_type}
|
||||
过期时间: {expires_at}
|
||||
|
||||
请重新登录同步:
|
||||
{login_url}
|
||||
|
||||
配置信息(点击复制):
|
||||
```json
|
||||
{config_json}
|
||||
```"""
|
||||
|
||||
print(message)
|
||||
|
||||
|
||||
def send_expiring_soon_notification(tag, data_type, remaining_days, login_url):
|
||||
"""发送即将过期通知"""
|
||||
message = f"""⚠️ 数据即将过期
|
||||
|
||||
Tag: {tag}
|
||||
类型: {data_type}
|
||||
剩余时间: {remaining_days:.1f} 天
|
||||
|
||||
建议重新登录同步:
|
||||
{login_url}"""
|
||||
|
||||
print(message)
|
||||
|
||||
|
||||
def check_expiry():
|
||||
"""检查所有数据的过期状态"""
|
||||
index = load_index()
|
||||
now = datetime.now()
|
||||
|
||||
expired_count = 0
|
||||
expiring_soon_count = 0
|
||||
|
||||
for item in index.get("tags", []):
|
||||
tag = item.get("tag")
|
||||
data_type = item.get("type", "unknown")
|
||||
expires_at_str = item.get("expires_at")
|
||||
filename = item.get("file")
|
||||
|
||||
if not expires_at_str or not filename:
|
||||
continue
|
||||
|
||||
# 解析过期时间
|
||||
expires_at = parse_datetime(expires_at_str)
|
||||
if not expires_at:
|
||||
continue
|
||||
|
||||
# 加载数据获取登录链接
|
||||
data = load_data(filename)
|
||||
login_url = "http://47.122.126.244:5001"
|
||||
if data and data.get("metadata"):
|
||||
login_url = data["metadata"].get("login_url", login_url)
|
||||
|
||||
# 计算剩余时间
|
||||
time_diff = expires_at - now
|
||||
remaining_seconds = time_diff.total_seconds()
|
||||
remaining_days = remaining_seconds / 86400
|
||||
|
||||
# 检查状态
|
||||
if remaining_seconds < 0:
|
||||
# 已过期
|
||||
expired_count += 1
|
||||
send_expired_notification(tag, data_type, expires_at_str, login_url)
|
||||
elif remaining_days < 2:
|
||||
# 即将过期(<2天)
|
||||
expiring_soon_count += 1
|
||||
send_expiring_soon_notification(tag, data_type, remaining_days, login_url)
|
||||
|
||||
# 发送汇总(仅当有问题时)
|
||||
if expired_count > 0 or expiring_soon_count > 0:
|
||||
summary = f"\n📊 检查完成\n\n"
|
||||
if expired_count > 0:
|
||||
summary += f"❌ 已过期: {expired_count} 个\n"
|
||||
if expiring_soon_count > 0:
|
||||
summary += f"⚠️ 即将过期: {expiring_soon_count} 个\n"
|
||||
print(summary)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
check_expiry()
|
||||
Executable
+85
@@ -0,0 +1,85 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
清理过期数据脚本
|
||||
删除已过期超过指定天数的数据
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
DATA_DIR = Path("/home/hermes/projects/data-sync-service/data")
|
||||
DELETE_AFTER_DAYS = 30 # 过期后保留天数
|
||||
|
||||
|
||||
def parse_datetime(dt_str):
|
||||
"""解析时间字符串"""
|
||||
try:
|
||||
return datetime.strptime(dt_str, "%Y-%m-%d %H:%M:%S")
|
||||
except:
|
||||
return None
|
||||
|
||||
|
||||
def clean_expired():
|
||||
"""清理过期数据"""
|
||||
|
||||
index_file = DATA_DIR / "index.json"
|
||||
|
||||
if not index_file.exists():
|
||||
print("⚠️ 索引文件不存在")
|
||||
return
|
||||
|
||||
with open(index_file, 'r', encoding='utf-8') as f:
|
||||
index = json.load(f)
|
||||
|
||||
now = datetime.now()
|
||||
threshold = now - timedelta(days=DELETE_AFTER_DAYS)
|
||||
|
||||
deleted_count = 0
|
||||
new_tags = []
|
||||
|
||||
for item in index.get("tags", []):
|
||||
tag = item.get("tag")
|
||||
expires_at_str = item.get("expires_at")
|
||||
filename = item.get("file")
|
||||
|
||||
if not expires_at_str:
|
||||
# 无过期时间,保留
|
||||
new_tags.append(item)
|
||||
continue
|
||||
|
||||
expires_at = parse_datetime(expires_at_str)
|
||||
if not expires_at:
|
||||
# 解析失败,保留
|
||||
new_tags.append(item)
|
||||
continue
|
||||
|
||||
# 检查是否过期超过阈值
|
||||
if expires_at < threshold:
|
||||
print(f"🗑️ 删除过期数据: {tag} (过期于 {expires_at_str})")
|
||||
|
||||
# 删除数据文件
|
||||
filepath = DATA_DIR / filename
|
||||
if filepath.exists():
|
||||
os.remove(filepath)
|
||||
|
||||
deleted_count += 1
|
||||
else:
|
||||
new_tags.append(item)
|
||||
|
||||
# 更新索引
|
||||
index["tags"] = new_tags
|
||||
with open(index_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(index, f, indent=2, ensure_ascii=False)
|
||||
|
||||
print(f"\n✅ 清理完成: 删除 {deleted_count} 个过期数据")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("=" * 60)
|
||||
print(f"清理过期数据 (过期超过 {DELETE_AFTER_DAYS} 天)")
|
||||
print("=" * 60)
|
||||
|
||||
clean_expired()
|
||||
@@ -0,0 +1,82 @@
|
||||
#!/bin/bash
|
||||
# 项目信息展示脚本
|
||||
|
||||
echo "================================================================"
|
||||
echo " 通用数据同步服务 (Generic Data Sync Service) v2.0"
|
||||
echo "================================================================"
|
||||
echo ""
|
||||
echo "[项目] 路径: /home/hermes/projects/data-sync-service"
|
||||
echo "[服务] 地址: http://47.122.126.244:5001"
|
||||
echo "[密码] admin***(完整密码见配置文件 nginx/nginx.conf)"
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo " 服务状态"
|
||||
echo "================================================================"
|
||||
echo ""
|
||||
|
||||
# 检查服务状态
|
||||
if docker ps | grep -q data-sync-service; then
|
||||
echo "[OK] Docker 容器: 运行中"
|
||||
else
|
||||
echo "[X] Docker 容器: 未运行"
|
||||
fi
|
||||
|
||||
# 检查端口
|
||||
if netstat -tuln 2>/dev/null | grep -q ":5001 "; then
|
||||
echo "[OK] 端口 5001: 监听中"
|
||||
else
|
||||
echo "[X] 端口 5001: 未监听"
|
||||
fi
|
||||
|
||||
# 健康检查
|
||||
if curl -s http://localhost:5001/health | grep -q "ok"; then
|
||||
echo "[OK] 健康检查: 通过"
|
||||
else
|
||||
echo "[X] 健康检查: 失败"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo " 资源占用"
|
||||
echo "================================================================"
|
||||
echo ""
|
||||
|
||||
docker stats --no-stream data-sync-service 2>/dev/null || echo "无法获取容器统计信息"
|
||||
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo " 数据统计"
|
||||
echo "================================================================"
|
||||
echo ""
|
||||
|
||||
# 数据文件数量
|
||||
DATA_COUNT=$(ls -1 data/*.json 2>/dev/null | grep -v index.json | wc -l)
|
||||
echo "[数据] 文件数: $DATA_COUNT"
|
||||
|
||||
# 索引信息
|
||||
if [ -f "data/index.json" ]; then
|
||||
TAG_COUNT=$(python3 -c "import json; print(len(json.load(open('data/index.json'))['tags']))" 2>/dev/null || echo "0")
|
||||
echo "[标签] Tag 数量: $TAG_COUNT"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo " 快速命令"
|
||||
echo "================================================================"
|
||||
echo ""
|
||||
echo "[日志] 查看日志:"
|
||||
echo " docker-compose logs -f"
|
||||
echo ""
|
||||
echo "[重启] 重启服务:"
|
||||
echo " docker-compose restart"
|
||||
echo ""
|
||||
echo "[测试] 测试 API:"
|
||||
echo " bash scripts/test_api.sh"
|
||||
echo ""
|
||||
echo "[过期] 检查过期:"
|
||||
echo " python3 scripts/check_expiry.py"
|
||||
echo ""
|
||||
echo "[文档] 查看文档:"
|
||||
echo " cat README.md"
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
数据迁移脚本 - 从旧的 cookie-receiver 迁移到新服务
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timedelta
|
||||
import shutil
|
||||
|
||||
OLD_DIR = Path('/home/hermes/projects/cookie-receiver/data')
|
||||
NEW_DIR = Path('/home/hermes/projects/data-sync-service/data')
|
||||
INDEX_FILE = NEW_DIR / 'index.json'
|
||||
|
||||
|
||||
def migrate():
|
||||
"""执行迁移"""
|
||||
print("🔄 开始迁移数据...\n")
|
||||
|
||||
# 确保新目录存在
|
||||
NEW_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 初始化索引
|
||||
if INDEX_FILE.exists():
|
||||
with open(INDEX_FILE, 'r', encoding='utf-8') as f:
|
||||
index = json.load(f)
|
||||
else:
|
||||
index = {"version": "1.0.0", "tags": {}}
|
||||
|
||||
# 迁移通义听悟数据
|
||||
old_file = OLD_DIR / 'tingwu_cookies.json'
|
||||
if old_file.exists():
|
||||
print(f"📁 发现旧数据: {old_file}")
|
||||
|
||||
with open(old_file, 'r', encoding='utf-8') as f:
|
||||
old_data = json.load(f)
|
||||
|
||||
# 提取账号信息
|
||||
account = old_data.get('account', 'ykaayk@qq.com')
|
||||
tag = f"tingwu:{account}"
|
||||
|
||||
# 转换为新格式
|
||||
now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
||||
expires_at = (datetime.now() + timedelta(days=7)).strftime('%Y-%m-%d %H:%M:%S')
|
||||
|
||||
new_data = {
|
||||
'tag': tag,
|
||||
'type': 'cookie',
|
||||
'data': {
|
||||
'cookies': old_data.get('cookies', []),
|
||||
'timestamp': old_data.get('timestamp', now)
|
||||
},
|
||||
'metadata': {
|
||||
'created_at': old_data.get('timestamp', now),
|
||||
'expires_in': 7 * 24 * 3600,
|
||||
'expires_at': expires_at,
|
||||
'note': '从旧 cookie-receiver 迁移',
|
||||
'login_url': 'https://tingwu.aliyun.com/'
|
||||
}
|
||||
}
|
||||
|
||||
# 保存新数据
|
||||
new_filename = 'tingwu_ykaayk_qq_com.json'
|
||||
new_filepath = NEW_DIR / new_filename
|
||||
|
||||
with open(new_filepath, 'w', encoding='utf-8') as f:
|
||||
json.dump(new_data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
print(f"✅ 已保存: {new_filepath}")
|
||||
|
||||
# 更新索引
|
||||
index['tags'][tag] = {
|
||||
'type': 'cookie',
|
||||
'file': new_filename,
|
||||
'created_at': now,
|
||||
'expires_at': expires_at,
|
||||
'last_accessed': now
|
||||
}
|
||||
|
||||
print(f"✅ 已更新索引: {tag}")
|
||||
|
||||
else:
|
||||
print(f"⚠️ 未找到旧数据文件: {old_file}")
|
||||
|
||||
# 保存索引
|
||||
with open(INDEX_FILE, 'w', encoding='utf-8') as f:
|
||||
json.dump(index, f, indent=2, ensure_ascii=False)
|
||||
|
||||
print(f"\n✅ 迁移完成!")
|
||||
print(f" 新数据目录: {NEW_DIR}")
|
||||
print(f" 索引文件: {INDEX_FILE}")
|
||||
print(f" 共迁移: {len(index['tags'])} 个数据")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
migrate()
|
||||
Executable
+140
@@ -0,0 +1,140 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
数据迁移脚本 - 从 cookie-receiver 迁移到 data-sync-service
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
OLD_DATA_DIR = Path("/home/hermes/projects/cookie-receiver/data")
|
||||
NEW_DATA_DIR = Path("/home/hermes/projects/data-sync-service/data")
|
||||
|
||||
|
||||
def migrate_tingwu_cookies():
|
||||
"""迁移通义听悟 Cookie 数据"""
|
||||
|
||||
old_file = OLD_DATA_DIR / "tingwu_cookies.json"
|
||||
|
||||
if not old_file.exists():
|
||||
print("⚠️ 旧数据文件不存在,跳过迁移")
|
||||
return
|
||||
|
||||
print(f"📂 读取旧数据: {old_file}")
|
||||
|
||||
with open(old_file, 'r', encoding='utf-8') as f:
|
||||
old_data = json.load(f)
|
||||
|
||||
# 提取账号信息(如果存在)
|
||||
account = old_data.get("account", "ykaayk@qq.com")
|
||||
|
||||
# 构建新的数据格式
|
||||
new_data = {
|
||||
"tag": f"tingwu:{account}",
|
||||
"type": "cookie",
|
||||
"data": old_data,
|
||||
"metadata": {
|
||||
"created_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"updated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"expires_in": 604800, # 7天
|
||||
"note": "从 cookie-receiver 迁移",
|
||||
"login_url": "https://tingwu.aliyun.com/"
|
||||
}
|
||||
}
|
||||
|
||||
# 生成新文件名
|
||||
filename = f"tingwu_{account.replace('@', '_').replace('.', '_')}.json"
|
||||
new_file = NEW_DATA_DIR / filename
|
||||
|
||||
print(f"💾 保存新数据: {new_file}")
|
||||
|
||||
with open(new_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(new_data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
# 更新索引
|
||||
update_index(new_data, filename)
|
||||
|
||||
print(f"✅ 迁移成功: tingwu_cookies.json → {filename}")
|
||||
|
||||
|
||||
def update_index(data, filename):
|
||||
"""更新索引文件"""
|
||||
|
||||
index_file = NEW_DATA_DIR / "index.json"
|
||||
|
||||
# 读取现有索引
|
||||
if index_file.exists():
|
||||
with open(index_file, 'r', encoding='utf-8') as f:
|
||||
index = json.load(f)
|
||||
else:
|
||||
index = {"tags": []}
|
||||
|
||||
# 添加新 tag
|
||||
tag_entry = {
|
||||
"tag": data["tag"],
|
||||
"type": data["type"],
|
||||
"file": filename,
|
||||
"created_at": data["metadata"]["created_at"],
|
||||
"updated_at": data["metadata"]["updated_at"],
|
||||
"expires_at": None
|
||||
}
|
||||
|
||||
# 计算过期时间
|
||||
if data["metadata"].get("expires_in"):
|
||||
from datetime import datetime, timedelta
|
||||
created = datetime.strptime(data["metadata"]["created_at"], "%Y-%m-%d %H:%M:%S")
|
||||
expires = created + timedelta(seconds=data["metadata"]["expires_in"])
|
||||
tag_entry["expires_at"] = expires.strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
# 检查是否已存在
|
||||
existing = False
|
||||
for i, item in enumerate(index["tags"]):
|
||||
if item["tag"] == data["tag"]:
|
||||
index["tags"][i] = tag_entry
|
||||
existing = True
|
||||
break
|
||||
|
||||
if not existing:
|
||||
index["tags"].append(tag_entry)
|
||||
|
||||
# 保存索引
|
||||
with open(index_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(index, f, indent=2, ensure_ascii=False)
|
||||
|
||||
print(f"📋 索引已更新")
|
||||
|
||||
|
||||
def main():
|
||||
"""主函数"""
|
||||
|
||||
print("=" * 60)
|
||||
print("数据迁移工具 - cookie-receiver → data-sync-service")
|
||||
print("=" * 60)
|
||||
|
||||
# 检查目录
|
||||
if not OLD_DATA_DIR.exists():
|
||||
print(f"❌ 旧数据目录不存在: {OLD_DATA_DIR}")
|
||||
return
|
||||
|
||||
# 创建新数据目录
|
||||
NEW_DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
print(f"✅ 新数据目录: {NEW_DATA_DIR}")
|
||||
|
||||
# 迁移数据
|
||||
print("\n📦 开始迁移...")
|
||||
migrate_tingwu_cookies()
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("✅ 迁移完成!")
|
||||
print("\n下一步:")
|
||||
print("1. 停止旧服务: cd /home/hermes/projects/cookie-receiver && docker-compose down")
|
||||
print("2. 启动新服务: cd /home/hermes/projects/data-sync-service && docker-compose up -d")
|
||||
print("3. 验证数据: curl http://localhost:5001/api/data")
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+132
@@ -0,0 +1,132 @@
|
||||
#!/bin/bash
|
||||
# 通用数据同步服务 - 完整测试脚本
|
||||
|
||||
echo "============================================================"
|
||||
echo "通用数据同步服务 - API 测试"
|
||||
echo "============================================================"
|
||||
|
||||
SERVER="http://localhost:5001"
|
||||
PASSWORD="admin123123123"
|
||||
|
||||
# 颜色定义
|
||||
GREEN='\033[0;32m'
|
||||
RED='\033[0;31m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
test_count=0
|
||||
pass_count=0
|
||||
|
||||
# 测试函数
|
||||
test_api() {
|
||||
test_count=$((test_count + 1))
|
||||
local name="$1"
|
||||
local expected="$2"
|
||||
shift 2
|
||||
local output
|
||||
output=$("$@" 2>&1)
|
||||
local result=$?
|
||||
|
||||
if [ $result -eq 0 ] && echo "$output" | grep -q "$expected"; then
|
||||
echo -e "${GREEN}✓${NC} 测试 $test_count: $name"
|
||||
pass_count=$((pass_count + 1))
|
||||
return 0
|
||||
else
|
||||
echo -e "${RED}✗${NC} 测试 $test_count: $name"
|
||||
echo " 预期: $expected"
|
||||
echo " 实际: $output"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
echo ""
|
||||
echo "1️⃣ 健康检查"
|
||||
test_api "健康检查" "ok" curl -s $SERVER/health
|
||||
|
||||
echo ""
|
||||
echo "2️⃣ 列出所有数据(初始状态)"
|
||||
test_api "列出所有数据" "tags" curl -s $SERVER/api/data
|
||||
|
||||
echo ""
|
||||
echo "3️⃣ 上传 Cookie 数据"
|
||||
test_api "上传 Cookie" "success" curl -s -X POST $SERVER/api/data \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"password\": \"$PASSWORD\",
|
||||
\"tag\": \"test:user1\",
|
||||
\"type\": \"cookie\",
|
||||
\"data\": {
|
||||
\"cookies\": [{\"name\": \"session\", \"value\": \"test123\"}]
|
||||
},
|
||||
\"metadata\": {
|
||||
\"expires_in\": 604800,
|
||||
\"note\": \"测试用户1\",
|
||||
\"login_url\": \"https://example.com\"
|
||||
}
|
||||
}"
|
||||
|
||||
echo ""
|
||||
echo "4️⃣ 上传 Token 数据"
|
||||
test_api "上传 Token" "success" curl -s -X POST $SERVER/api/data \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"password\": \"$PASSWORD\",
|
||||
\"tag\": \"api:service\",
|
||||
\"type\": \"token\",
|
||||
\"data\": {
|
||||
\"token\": \"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9...\",
|
||||
\"token_type\": \"Bearer\"
|
||||
},
|
||||
\"metadata\": {
|
||||
\"expires_in\": 2592000,
|
||||
\"note\": \"API 服务账号\"
|
||||
}
|
||||
}"
|
||||
|
||||
echo ""
|
||||
echo "5️⃣ 获取数据(有密码)"
|
||||
test_api "获取数据" "test:user1" curl -s -H "X-Password: $PASSWORD" $SERVER/api/data/test:user1
|
||||
|
||||
echo ""
|
||||
echo "6️⃣ 获取数据(无密码,应失败)"
|
||||
test_api "无密码获取" "Invalid password" curl -s $SERVER/api/data/test:user1
|
||||
|
||||
echo ""
|
||||
echo "7️⃣ 列出所有数据(应有2条)"
|
||||
test_api "列出数据" "test:user1" curl -s $SERVER/api/data
|
||||
|
||||
echo ""
|
||||
echo "8️⃣ 一键复制配置"
|
||||
test_api "复制配置" "copy_text" curl -s $SERVER/api/data/copy/test:user1
|
||||
|
||||
echo ""
|
||||
echo "9️⃣ 删除数据"
|
||||
test_api "删除数据" "success" curl -s -X DELETE -H "X-Password: $PASSWORD" $SERVER/api/data/test:user1
|
||||
|
||||
echo ""
|
||||
echo "🔟 验证删除(应返回404)"
|
||||
test_api "验证删除" "not found" curl -s -H "X-Password: $PASSWORD" $SERVER/api/data/test:user1
|
||||
|
||||
echo ""
|
||||
echo "1️⃣1️⃣ 错误密码测试"
|
||||
test_api "错误密码" "Invalid password" curl -s -X POST $SERVER/api/data \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{
|
||||
\"password\": \"wrongpassword\",
|
||||
\"tag\": \"test:fail\",
|
||||
\"type\": \"cookie\",
|
||||
\"data\": {}
|
||||
}"
|
||||
|
||||
echo ""
|
||||
echo "============================================================"
|
||||
echo "测试结果: $pass_count / $test_count 通过"
|
||||
echo "============================================================"
|
||||
|
||||
if [ $pass_count -eq $test_count ]; then
|
||||
echo -e "${GREEN}✓ 所有测试通过!${NC}"
|
||||
exit 0
|
||||
else
|
||||
echo -e "${RED}✗ 部分测试失败${NC}"
|
||||
exit 1
|
||||
fi
|
||||
Executable
+185
@@ -0,0 +1,185 @@
|
||||
"""
|
||||
通用数据同步服务 - 工具函数库
|
||||
"""
|
||||
|
||||
import json
|
||||
import requests
|
||||
from pathlib import Path
|
||||
from typing import Optional, Dict, Any, List
|
||||
|
||||
|
||||
SERVER_URL = "http://47.122.126.244:5001"
|
||||
PASSWORD = "admin123123123"
|
||||
|
||||
|
||||
def upload_data(tag: str, data_type: str, data: Dict[Any, Any],
|
||||
expires_in: Optional[int] = None,
|
||||
note: Optional[str] = None,
|
||||
login_url: Optional[str] = None) -> bool:
|
||||
"""
|
||||
上传数据到同步服务
|
||||
|
||||
Args:
|
||||
tag: 数据标签(格式: 项目名:账号名)
|
||||
data_type: 数据类型 (cookie/token/credential/session/custom)
|
||||
data: 数据内容
|
||||
expires_in: 过期时间(秒)
|
||||
note: 备注
|
||||
login_url: 登录链接
|
||||
|
||||
Returns:
|
||||
bool: 上传成功返回 True
|
||||
"""
|
||||
payload = {
|
||||
"password": PASSWORD,
|
||||
"tag": tag,
|
||||
"type": data_type,
|
||||
"data": data,
|
||||
"metadata": {}
|
||||
}
|
||||
|
||||
if expires_in:
|
||||
payload["metadata"]["expires_in"] = expires_in
|
||||
if note:
|
||||
payload["metadata"]["note"] = note
|
||||
if login_url:
|
||||
payload["metadata"]["login_url"] = login_url
|
||||
|
||||
try:
|
||||
response = requests.post(
|
||||
f"{SERVER_URL}/api/data",
|
||||
json=payload,
|
||||
timeout=10
|
||||
)
|
||||
return response.status_code == 200
|
||||
except Exception as e:
|
||||
print(f"⚠️ 上传失败: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def get_data(tag: str) -> Optional[Dict[Any, Any]]:
|
||||
"""
|
||||
获取数据
|
||||
|
||||
Args:
|
||||
tag: 数据标签
|
||||
|
||||
Returns:
|
||||
Dict: 数据内容,失败返回 None
|
||||
"""
|
||||
try:
|
||||
response = requests.get(
|
||||
f"{SERVER_URL}/api/data/{tag}",
|
||||
headers={"X-Password": PASSWORD},
|
||||
timeout=10
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
return response.json()
|
||||
return None
|
||||
except Exception as e:
|
||||
print(f"⚠️ 获取数据失败: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def delete_data(tag: str) -> bool:
|
||||
"""
|
||||
删除数据
|
||||
|
||||
Args:
|
||||
tag: 数据标签
|
||||
|
||||
Returns:
|
||||
bool: 删除成功返回 True
|
||||
"""
|
||||
try:
|
||||
response = requests.delete(
|
||||
f"{SERVER_URL}/api/data/{tag}",
|
||||
headers={"X-Password": PASSWORD},
|
||||
timeout=10
|
||||
)
|
||||
return response.status_code == 200
|
||||
except Exception as e:
|
||||
print(f"⚠️ 删除失败: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def list_all_tags() -> List[Dict[str, Any]]:
|
||||
"""
|
||||
列出所有 tag
|
||||
|
||||
Returns:
|
||||
List: tag 列表
|
||||
"""
|
||||
try:
|
||||
response = requests.get(
|
||||
f"{SERVER_URL}/api/data",
|
||||
timeout=10
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
return data.get("tags", [])
|
||||
return []
|
||||
except Exception as e:
|
||||
print(f"⚠️ 获取列表失败: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def get_copy_config(tag: str) -> Optional[str]:
|
||||
"""
|
||||
获取一键复制配置
|
||||
|
||||
Args:
|
||||
tag: 数据标签
|
||||
|
||||
Returns:
|
||||
str: 配置文本
|
||||
"""
|
||||
try:
|
||||
response = requests.get(
|
||||
f"{SERVER_URL}/api/data/copy/{tag}",
|
||||
timeout=10
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
return data.get("copy_text")
|
||||
return None
|
||||
except Exception as e:
|
||||
print(f"⚠️ 获取配置失败: {e}")
|
||||
return None
|
||||
|
||||
|
||||
# 便捷函数:从同步服务加载 Cookie 数据
|
||||
def load_cookies_from_sync(tag: str) -> Optional[List[Dict[str, Any]]]:
|
||||
"""
|
||||
从同步服务加载 Cookie 数据
|
||||
|
||||
Args:
|
||||
tag: 数据标签
|
||||
|
||||
Returns:
|
||||
List: Cookie 列表,失败返回 None
|
||||
"""
|
||||
data = get_data(tag)
|
||||
if data and data.get("type") == "cookie":
|
||||
return data.get("data", {}).get("cookies")
|
||||
return None
|
||||
|
||||
|
||||
# 便捷函数:从同步服务加载 Token 数据
|
||||
def load_token_from_sync(tag: str) -> Optional[str]:
|
||||
"""
|
||||
从同步服务加载 Token 数据
|
||||
|
||||
Args:
|
||||
tag: 数据标签
|
||||
|
||||
Returns:
|
||||
str: Token 字符串,失败返回 None
|
||||
"""
|
||||
data = get_data(tag)
|
||||
if data and data.get("type") == "token":
|
||||
return data.get("data", {}).get("token")
|
||||
return None
|
||||
Reference in New Issue
Block a user