chore: 以本地为准，上传全部并替换 GitHub
This commit is contained in:
卡若
297
app/api/data-sources/route.ts
Normal file
297
app/api/data-sources/route.ts
Normal file
@@ -0,0 +1,297 @@
|
||||
/**
|
||||
* 数据源管理 API
|
||||
* 提供数据源列表、连接状态检测、同步管理等功能
|
||||
*/
|
||||
|
||||
import { NextRequest, NextResponse } from 'next/server'
|
||||
import { getMongoClient, getDatabaseStats } from '@/lib/mongodb'
|
||||
|
||||
// 数据源接口定义
|
||||
interface DataSource {
|
||||
id: string
|
||||
name: string
|
||||
nameCn: string // 中文名称
|
||||
description: string // 功能描述
|
||||
type: 'mongodb' | 'mysql' | 'api' | 'webhook'
|
||||
status: 'connected' | 'disconnected' | 'warning'
|
||||
host?: string
|
||||
database?: string
|
||||
endpoint?: string
|
||||
lastSync: string
|
||||
recordCount: number
|
||||
syncFrequency: string
|
||||
collections?: number
|
||||
tables?: number
|
||||
latency?: number
|
||||
dataCategory?: string // 数据分类
|
||||
targetCollection?: string // 目标集合（数据中台分配）
|
||||
}
|
||||
|
||||
// 数据库中文名称和描述映射
|
||||
const DB_DESCRIPTIONS: Record<string, { nameCn: string; description: string; category: string }> = {
|
||||
'KR': { nameCn: '核心用户库', description: '用户估值、RFM评分、统一画像', category: '用户画像' },
|
||||
'KR_KR': { nameCn: '扩展用户库', description: '多源用户数据汇聚', category: '用户数据' },
|
||||
'KR_Linkedln': { nameCn: '领英数据库', description: '职业社交平台用户数据', category: '社交数据' },
|
||||
'KR_京东': { nameCn: '京东用户库', description: '京东电商用户消费数据', category: '电商数据' },
|
||||
'KR_人才库': { nameCn: '人才数据库', description: '招聘平台人才信息', category: '人力资源' },
|
||||
'KR_企业': { nameCn: '企业信息库', description: '企业工商注册信息', category: '企业数据' },
|
||||
'KR_企业名录': { nameCn: '企业名录', description: '企业联系方式和经营信息', category: '企业数据' },
|
||||
'KR_卡若私域': { nameCn: '卡若私域库', description: '私域运营用户数据', category: '私域数据' },
|
||||
'KR_商城': { nameCn: '商城用户库', description: '电商平台用户交易数据', category: '电商数据' },
|
||||
'KR_国外': { nameCn: '海外用户库', description: '海外平台用户数据', category: '国际数据' },
|
||||
'KR_存客宝': { nameCn: '存客宝CRM', description: '私域CRM用户资产数据', category: '私域数据' },
|
||||
'KR_存客宝_四表重构KR_KR版': { nameCn: '存客宝重构版', description: '存客宝数据统一重构', category: '私域数据' },
|
||||
'KR_微博': { nameCn: '微博用户库', description: '微博UID与手机号关联', category: '社交数据' },
|
||||
'KR_快递': { nameCn: '快递信息库', description: '快递收发地址信息', category: '物流数据' },
|
||||
'KR_户口': { nameCn: '户籍信息库', description: '户籍地址信息', category: '身份数据' },
|
||||
'KR_手机': { nameCn: '手机号库', description: '手机号归属地和运营商', category: '基础数据' },
|
||||
'KR_投资': { nameCn: '投资信息库', description: '投资理财用户数据', category: '金融数据' },
|
||||
'KR_淘宝': { nameCn: '淘宝用户库', description: '淘宝电商用户数据', category: '电商数据' },
|
||||
'KR_游戏': { nameCn: '游戏用户库', description: '游戏平台用户数据', category: '娱乐数据' },
|
||||
'KR_点了码': { nameCn: '点了码商户库', description: '点了码商户和用户统一视图', category: '商业数据' },
|
||||
'KR_腾讯': { nameCn: '腾讯社交库', description: 'QQ号与手机号关联数据', category: '社交数据' },
|
||||
'KR_酒店': { nameCn: '酒店住宿库', description: '酒店入住记录信息', category: '消费数据' },
|
||||
'KR_顺丰': { nameCn: '顺丰快递库', description: '顺丰快递收发信息', category: '物流数据' },
|
||||
'KR_魔兽世界': { nameCn: '魔兽世界库', description: '魔兽世界玩家数据', category: '游戏数据' },
|
||||
}
|
||||
|
||||
// 获取 MongoDB 数据源列表
|
||||
async function getMongoDBSources(): Promise<DataSource[]> {
|
||||
try {
|
||||
const client = await getMongoClient()
|
||||
const dbList = await client.db().admin().listDatabases()
|
||||
const sources: DataSource[] = []
|
||||
|
||||
for (const dbInfo of dbList.databases) {
|
||||
if (dbInfo.name.startsWith('KR')) {
|
||||
const db = client.db(dbInfo.name)
|
||||
const collections = await db.listCollections().toArray()
|
||||
|
||||
let totalDocs = 0
|
||||
for (const coll of collections) {
|
||||
const count = await db.collection(coll.name).estimatedDocumentCount()
|
||||
totalDocs += count
|
||||
}
|
||||
|
||||
// 获取中文名称和描述
|
||||
const dbMeta = DB_DESCRIPTIONS[dbInfo.name] || {
|
||||
nameCn: dbInfo.name,
|
||||
description: `${dbInfo.name} 数据库`,
|
||||
category: '其他'
|
||||
}
|
||||
|
||||
sources.push({
|
||||
id: `mongo_${dbInfo.name}`,
|
||||
name: dbInfo.name,
|
||||
nameCn: dbMeta.nameCn,
|
||||
description: dbMeta.description,
|
||||
type: 'mongodb',
|
||||
status: 'connected',
|
||||
host: process.env.MONGODB_URI?.split('@')[1]?.split('/')[0] || 'localhost:27017',
|
||||
database: dbInfo.name,
|
||||
lastSync: '实时',
|
||||
recordCount: totalDocs,
|
||||
syncFrequency: '实时',
|
||||
collections: collections.length,
|
||||
latency: 0,
|
||||
dataCategory: dbMeta.category,
|
||||
targetCollection: 'KR.用户估值' // 默认目标集合
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return sources
|
||||
} catch (error) {
|
||||
console.error('获取 MongoDB 数据源失败:', error)
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
// 检测数据源连接状态
|
||||
async function checkConnectionStatus(): Promise<{
|
||||
mongodb: { connected: boolean; latency: number }
|
||||
mysql: { connected: boolean; latency: number }
|
||||
}> {
|
||||
const startTime = Date.now()
|
||||
let mongoConnected = false
|
||||
let mongoLatency = 0
|
||||
|
||||
try {
|
||||
const client = await getMongoClient()
|
||||
await client.db().admin().ping()
|
||||
mongoConnected = true
|
||||
mongoLatency = Date.now() - startTime
|
||||
} catch {
|
||||
mongoLatency = Date.now() - startTime
|
||||
}
|
||||
|
||||
return {
|
||||
mongodb: { connected: mongoConnected, latency: mongoLatency },
|
||||
mysql: { connected: false, latency: 0 } // MySQL 暂未实现
|
||||
}
|
||||
}
|
||||
|
||||
// GET: 获取数据源列表
|
||||
export async function GET(request: NextRequest) {
|
||||
const { searchParams } = new URL(request.url)
|
||||
const action = searchParams.get('action')
|
||||
|
||||
try {
|
||||
// 检查连接状态
|
||||
if (action === 'status') {
|
||||
const status = await checkConnectionStatus()
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
status
|
||||
})
|
||||
}
|
||||
|
||||
// 获取数据库统计
|
||||
if (action === 'stats') {
|
||||
const stats = await getDatabaseStats()
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
stats
|
||||
})
|
||||
}
|
||||
|
||||
// 获取完整数据源列表
|
||||
const mongoSources = await getMongoDBSources()
|
||||
|
||||
// 计算统计信息
|
||||
const stats = await getDatabaseStats()
|
||||
|
||||
// 预定义的外部数据源（MySQL、API 等）
|
||||
const externalSources: DataSource[] = [
|
||||
{
|
||||
id: 'mysql_ckb',
|
||||
name: 'cunkebao_v3',
|
||||
nameCn: '存客宝MySQL主库',
|
||||
description: '存客宝CRM系统MySQL数据库，包含客户、订单、营销数据',
|
||||
type: 'mysql',
|
||||
status: 'warning',
|
||||
host: 'cdb-xxx.gz.tencentcdb.com:10050',
|
||||
database: 'cunkebao_v3',
|
||||
lastSync: '需要配置连接',
|
||||
recordCount: 0,
|
||||
syncFrequency: '增量同步',
|
||||
tables: 45,
|
||||
dataCategory: '私域数据',
|
||||
targetCollection: 'KR_存客宝.用户资产统一视图'
|
||||
},
|
||||
{
|
||||
id: 'mysql_dlm',
|
||||
name: 'dianlema',
|
||||
nameCn: '点了码MySQL库',
|
||||
description: '点了码商户系统，包含商户、用户、交易数据',
|
||||
type: 'mysql',
|
||||
status: 'warning',
|
||||
host: 'cdb-xxx.gz.tencentcdb.com:14413',
|
||||
database: 'dianlema',
|
||||
lastSync: '需要配置连接',
|
||||
recordCount: 0,
|
||||
syncFrequency: '增量同步',
|
||||
tables: 28,
|
||||
dataCategory: '商业数据',
|
||||
targetCollection: 'KR_点了码.用户资产统一视图'
|
||||
},
|
||||
{
|
||||
id: 'api_weibo',
|
||||
name: 'weibo_api',
|
||||
nameCn: '微博开放API',
|
||||
description: '微博开放平台API，获取用户公开信息和热点数据',
|
||||
type: 'api',
|
||||
status: 'disconnected',
|
||||
endpoint: 'https://api.weibo.com/2',
|
||||
lastSync: '未配置',
|
||||
recordCount: 0,
|
||||
syncFrequency: '按需调用',
|
||||
dataCategory: '社交数据',
|
||||
targetCollection: 'KR_微博.微博uid+手机'
|
||||
},
|
||||
{
|
||||
id: 'webhook_feishu',
|
||||
name: 'feishu_webhook',
|
||||
nameCn: '飞书机器人',
|
||||
description: '飞书机器人Webhook，接收对话消息并触发查询',
|
||||
type: 'webhook',
|
||||
status: 'connected',
|
||||
endpoint: '/api/feishu/webhook',
|
||||
lastSync: '实时',
|
||||
recordCount: 0,
|
||||
syncFrequency: '实时',
|
||||
dataCategory: '消息通道'
|
||||
}
|
||||
]
|
||||
|
||||
const allSources = [...mongoSources, ...externalSources]
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
sources: allSources,
|
||||
summary: {
|
||||
total: allSources.length,
|
||||
connected: allSources.filter(s => s.status === 'connected').length,
|
||||
warning: allSources.filter(s => s.status === 'warning').length,
|
||||
disconnected: allSources.filter(s => s.status === 'disconnected').length,
|
||||
totalRecords: stats.totalDocuments,
|
||||
totalSize: stats.totalSize,
|
||||
latency: stats.latency
|
||||
}
|
||||
})
|
||||
} catch (error: any) {
|
||||
console.error('数据源 API 错误:', error)
|
||||
return NextResponse.json({
|
||||
success: false,
|
||||
error: error.message,
|
||||
sources: [],
|
||||
summary: {
|
||||
total: 0,
|
||||
connected: 0,
|
||||
warning: 0,
|
||||
disconnected: 0,
|
||||
totalRecords: 0,
|
||||
totalSize: 0,
|
||||
latency: 0
|
||||
}
|
||||
}, { status: 500 })
|
||||
}
|
||||
}
|
||||
|
||||
// POST: 测试数据源连接
|
||||
export async function POST(request: NextRequest) {
|
||||
try {
|
||||
const body = await request.json()
|
||||
const { type, config } = body
|
||||
|
||||
if (type === 'mongodb') {
|
||||
const { MongoClient } = await import('mongodb')
|
||||
const uri = config.uri || `mongodb://${config.username}:${config.password}@${config.host}/?authSource=admin`
|
||||
|
||||
const client = new MongoClient(uri, {
|
||||
serverSelectionTimeoutMS: 5000
|
||||
})
|
||||
|
||||
await client.connect()
|
||||
await client.db().admin().ping()
|
||||
await client.close()
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
message: 'MongoDB 连接成功'
|
||||
})
|
||||
}
|
||||
|
||||
// 其他数据源类型暂未实现
|
||||
return NextResponse.json({
|
||||
success: false,
|
||||
message: `${type} 类型暂未支持`
|
||||
}, { status: 400 })
|
||||
|
||||
} catch (error: any) {
|
||||
return NextResponse.json({
|
||||
success: false,
|
||||
error: error.message
|
||||
}, { status: 500 })
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user