injectlib/urldb
1
0
Fork 0
mirror of https://github.com/ctwj/urldb.git synced 2025-11-25 03:15:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

update u

Browse Source
This commit is contained in:
ctwj
2025-11-24 01:06:47 +08:00
parent 8a79e7e87b
commit 4e3f9017ac
23 changed files with 4104 additions and 302 deletions
Download Patch File Download Diff File Expand all files Collapse all files

295
cmd/google-index/main.go Normal file
View File

@@ -0,0 +1,295 @@
package main
import (
"fmt"
"log"
"os"
"strings"
"time"
"github.com/ctwj/urldb/pkg/google"
)
func main() {
if len(os.Args) < 2 {
printUsage()
return
}
command := os.Args[1]
config := &google.Config{
CredentialsFile: "credentials.json",
SiteURL: "https://your-site.com/", // 替换为你的网站
TokenFile: "token.json",
}
client, err := google.NewClient(config)
if err != nil {
log.Fatalf("创建Google客户端失败: %v", err)
}
switch command {
case "inspect":
if len(os.Args) < 3 {
fmt.Println("请提供要检查的URL")
return
}
inspectSingleURL(client, os.Args[2])
case "batch":
if len(os.Args) < 3 {
fmt.Println("请提供包含URL列表的文件")
return
}
batchInspectURLs(client, os.Args[2])
case "sites":
listSites(client)
case "analytics":
getAnalytics(client)
case "sitemap":
if len(os.Args) < 3 {
fmt.Println("请提供网站地图URL")
return
}
submitSitemap(client, os.Args[2])
default:
fmt.Printf("未知命令: %s\n", command)
printUsage()
}
}
func printUsage() {
fmt.Println("Google Search Console API 工具")
fmt.Println()
fmt.Println("用法:")
fmt.Println(" google-index inspect <url> - 检查单个URL的索引状态")
fmt.Println(" google-index batch <file> - 批量检查URL状态")
fmt.Println(" google-index sites - 列出已验证的网站")
fmt.Println(" google-index analytics - 获取搜索分析数据")
fmt.Println(" google-index sitemap <url> - 提交网站地图")
fmt.Println()
fmt.Println("配置:")
fmt.Println(" - 创建 credentials.json 文件 (Google Cloud Console 下载)")
fmt.Println(" - 修改 config.SiteURL 为你的网站URL")
}
func inspectSingleURL(client *google.Client, url string) {
fmt.Printf("正在检查URL: %s\n", url)
result, err := client.InspectURL(url)
if err != nil {
log.Printf("检查失败: %v", err)
return
}
printResult(url, result)
}
func batchInspectURLs(client *google.Client, filename string) {
urls, err := readURLsFromFile(filename)
if err != nil {
log.Fatalf("读取URL文件失败: %v", err)
}
fmt.Printf("开始批量检查 %d 个URL...\n", len(urls))
results := make(chan struct {
url string
result *google.URLInspectionResult
err error
}, len(urls))
client.BatchInspectURL(urls, func(url string, result *google.URLInspectionResult, err error) {
results <- struct {
url string
result *google.URLInspectionResult
err error
}{url, result, err}
})
// 收集并打印结果
fmt.Println("\n检查结果:")
fmt.Println(strings.Repeat("-", 100))
fmt.Printf("%-50s %-15s %-15s %-20s\n", "URL", "索引状态", "移动友好", "最后抓取")
fmt.Println(strings.Repeat("-", 100))
for i := 0; i < len(urls); i++ {
res := <-results
if res.err != nil {
fmt.Printf("%-50s %-15s\n", truncate(res.url, 47), "ERROR")
continue
}
indexStatus := res.result.IndexStatusResult.IndexingState
mobileFriendly := "否"
if res.result.MobileUsabilityResult.MobileFriendly {
mobileFriendly = "是"
}
lastCrawl := res.result.IndexStatusResult.LastCrawled
if lastCrawl == "" {
lastCrawl = "未知"
}
fmt.Printf("%-50s %-15s %-15s %-20s\n",
truncate(res.url, 47), indexStatus, mobileFriendly, lastCrawl)
}
fmt.Println(strings.Repeat("-", 100))
}
func listSites(client *google.Client) {
sites, err := client.GetSites()
if err != nil {
log.Printf("获取网站列表失败: %v", err)
return
}
fmt.Println("已验证的网站:")
fmt.Println(strings.Repeat("-", 80))
fmt.Printf("%-50s %-15s %-15s\n", "网站URL", "权限级别", "验证状态")
fmt.Println(strings.Repeat("-", 80))
for _, site := range sites {
permissionLevel := string(site.PermissionLevel)
verified := "否"
if site.SiteUrl == client.SiteURL {
verified = "是"
}
fmt.Printf("%-50s %-15s %-15s\n",
truncate(site.SiteUrl, 47), permissionLevel, verified)
}
fmt.Println(strings.Repeat("-", 80))
}
func getAnalytics(client *google.Client) {
endDate := time.Now().Format("2006-01-02")
startDate := time.Now().AddDate(0, -1, 0).Format("2006-01-02") // 最近30天
fmt.Printf("获取搜索分析数据 (%s 到 %s)...\n", startDate, endDate)
analytics, err := client.GetSearchAnalytics(startDate, endDate)
if err != nil {
log.Printf("获取分析数据失败: %v", err)
return
}
// 计算总计数据
var totalClicks, totalImpressions float64
var totalPosition float64
for _, row := range analytics.Rows {
totalClicks += row.Clicks
totalImpressions += row.Impressions
totalPosition += row.Position
}
avgCTR := float64(0)
if totalImpressions > 0 {
avgCTR = float64(totalClicks) / float64(totalImpressions) * 100
}
avgPosition := float64(0)
if len(analytics.Rows) > 0 {
avgPosition = totalPosition / float64(len(analytics.Rows))
}
fmt.Println("\n搜索分析摘要:")
fmt.Println(strings.Repeat("-", 60))
fmt.Printf("总点击数: %.0f\n", totalClicks)
fmt.Printf("总展示次数: %.0f\n", totalImpressions)
fmt.Printf("平均点击率: %.2f%%\n", avgCTR)
fmt.Printf("平均排名: %.1f\n", avgPosition)
fmt.Println(strings.Repeat("-", 60))
// 显示前10个页面
if len(analytics.Rows) > 0 {
fmt.Println("\n热门页面 (前10):")
fmt.Printf("%-50s %-10s %-10s %-10s\n", "页面", "点击", "展示", "排名")
fmt.Println(strings.Repeat("-", 80))
maxRows := len(analytics.Rows)
if maxRows > 10 {
maxRows = 10
}
for i := 0; i < maxRows; i++ {
row := analytics.Rows[i]
fmt.Printf("%-50s %-10d %-10d %-10.1f\n",
truncate(row.Keys[0], 47), row.Clicks, row.Impressions, row.Position)
}
}
}
func submitSitemap(client *google.Client, sitemapURL string) {
fmt.Printf("正在提交网站地图: %s\n", sitemapURL)
err := client.SubmitSitemap(sitemapURL)
if err != nil {
log.Printf("提交网站地图失败: %v", err)
return
}
fmt.Println("网站地图提交成功!")
}
func readURLsFromFile(filename string) ([]string, error) {
data, err := os.ReadFile(filename)
if err != nil {
return nil, err
}
lines := strings.Split(string(data), "\n")
var urls []string
for _, line := range lines {
line = strings.TrimSpace(line)
if line != "" && !strings.HasPrefix(line, "#") {
urls = append(urls, line)
}
}
return urls, nil
}
func printResult(url string, result *google.URLInspectionResult) {
fmt.Println("\nURL检查结果:")
fmt.Println(strings.Repeat("=", 80))
fmt.Printf("URL: %s\n", url)
fmt.Println(strings.Repeat("-", 80))
fmt.Printf("索引状态: %s\n", result.IndexStatusResult.IndexingState)
if result.IndexStatusResult.LastCrawled != "" {
fmt.Printf("最后抓取: %s\n", result.IndexStatusResult.LastCrawled)
}
fmt.Printf("移动友好: %t\n", result.MobileUsabilityResult.MobileFriendly)
if len(result.RichResultsResult.Detected.Items) > 0 {
fmt.Println("富媒体结果:")
for _, item := range result.RichResultsResult.Detected.Items {
fmt.Printf(" - %s\n", item.RichResultType)
}
}
if len(result.IndexStatusResult.CrawlErrors) > 0 {
fmt.Println("抓取错误:")
for _, err := range result.IndexStatusResult.CrawlErrors {
fmt.Printf(" - %s\n", err.ErrorCode)
}
}
fmt.Println(strings.Repeat("=", 80))
}
func truncate(s string, length int) string {
if len(s) <= length {
return s
}
return s[:length-3] + "..."
}

36
db/converter/converter.go
View File

@@ -1,6 +1,7 @@
package converter
import (
"encoding/json"
"reflect"
"time"
@@ -325,3 +326,38 @@ func ToReadyResourceResponseList(resources []entity.ReadyResource) []dto.ReadyRe
// Key: req.Key,
// }
// }
// TaskToGoogleIndexTaskOutput 将Task实体转换为GoogleIndexTaskOutput
func TaskToGoogleIndexTaskOutput(task *entity.Task, stats map[string]int) dto.GoogleIndexTaskOutput {
output := dto.GoogleIndexTaskOutput{
ID: task.ID,
Name: task.Name,
Description: task.Description,
Type: string(task.Type),
Status: string(task.Status),
Progress: task.Progress,
TotalItems: stats["total"],
ProcessedItems: stats["completed"] + stats["failed"],
SuccessfulItems: stats["completed"],
FailedItems: stats["failed"],
PendingItems: stats["pending"],
ProcessingItems: stats["processing"],
IndexedURLs: task.IndexedURLs,
FailedURLs: task.FailedURLs,
ConfigID: task.ConfigID,
CreatedAt: task.CreatedAt,
UpdatedAt: task.UpdatedAt,
StartedAt: task.StartedAt,
CompletedAt: task.CompletedAt,
}
// 设置进度数据
if task.ProgressData != "" {
var progressData map[string]interface{}
if err := json.Unmarshal([]byte(task.ProgressData), &progressData); err == nil {
output.ProgressData = progressData
}
}
return output
}

3
db/converter/system_config_converter.go
View File

@@ -305,6 +305,7 @@ func SystemConfigToPublicResponse(configs []entity.SystemConfig) map[string]inte
entity.ConfigResponseFieldEnableRegister: true, // 默认开启注册功能
entity.ConfigResponseFieldThirdPartyStatsCode: "",
entity.ConfigResponseFieldWebsiteURL: "",
"google_site_verification_code": "",
}
// 将键值对转换为map，过滤掉敏感配置
@@ -362,6 +363,8 @@ func SystemConfigToPublicResponse(configs []entity.SystemConfig) map[string]inte
response["qr_code_style"] = config.Value
case entity.ConfigKeyWebsiteURL:
response[entity.ConfigResponseFieldWebsiteURL] = config.Value
case "google_site_verification_code":
response["google_site_verification_code"] = config.Value
case entity.ConfigKeyAutoProcessReadyResources:
if val, err := strconv.ParseBool(config.Value); err == nil {
response["auto_process_ready_resources"] = val

188
db/dto/google_index.go Normal file
View File

@@ -0,0 +1,188 @@
package dto
import (
"time"
)
// GoogleIndexConfigInput Google索引配置输入
type GoogleIndexConfigInput struct {
Group string `json:"group" binding:"required"`
Key string `json:"key" binding:"required"`
Value string `json:"value" binding:"required"`
Type string `json:"type" default:"string"`
}
// GoogleIndexConfigOutput Google索引配置输出
type GoogleIndexConfigOutput struct {
ID uint `json:"id"`
Group string `json:"group"`
Key string `json:"key"`
Value string `json:"value"`
Type string `json:"type"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
// GoogleIndexConfigGeneral 通用配置
type GoogleIndexConfigGeneral struct {
Enabled bool `json:"enabled" binding:"required"`
SiteURL string `json:"site_url" binding:"required"`
SiteName string `json:"site_name"`
}
// GoogleIndexConfigAuth 认证配置
type GoogleIndexConfigAuth struct {
CredentialsFile string `json:"credentials_file"`
ClientEmail string `json:"client_email"`
ClientID string `json:"client_id"`
PrivateKey string `json:"private_key"`
Token string `json:"token"`
}
// GoogleIndexConfigSchedule 调度配置
type GoogleIndexConfigSchedule struct {
CheckInterval int `json:"check_interval" binding:"required,min=1,max=1440"` // 检查间隔（分钟），1-24小时
BatchSize int `json:"batch_size" binding:"required,min=1,max=1000"` // 批处理大小
Concurrency int `json:"concurrency" binding:"required,min=1,max=10"` // 并发数
RetryAttempts int `json:"retry_attempts" binding:"required,min=0,max=10"` // 重试次数
RetryDelay int `json:"retry_delay" binding:"required,min=1,max=60"` // 重试延迟（秒）
}
// GoogleIndexConfigSitemap 网站地图配置
type GoogleIndexConfigSitemap struct {
AutoSitemap bool `json:"auto_sitemap"`
SitemapPath string `json:"sitemap_path" default:"/sitemap.xml"`
SitemapSchedule string `json:"sitemap_schedule" default:"@daily"` // cron表达式
}
// GoogleIndexTaskInput Google索引任务输入
type GoogleIndexTaskInput struct {
Title string `json:"title" binding:"required"`
Type string `json:"type" binding:"required"`
Description string `json:"description"`
URLs []string `json:"urls,omitempty"` // 用于URL索引检查任务
SitemapURL string `json:"sitemap_url,omitempty"` // 用于网站地图提交任务
ConfigID *uint `json:"config_id,omitempty"`
}
// GoogleIndexTaskOutput Google索引任务输出
type GoogleIndexTaskOutput struct {
ID uint `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
Type string `json:"type"`
Status string `json:"status"`
Progress float64 `json:"progress"`
TotalItems int `json:"total_items"`
ProcessedItems int `json:"processed_items"`
SuccessfulItems int `json:"successful_items"`
FailedItems int `json:"failed_items"`
PendingItems int `json:"pending_items"`
ProcessingItems int `json:"processing_items"`
IndexedURLs int `json:"indexed_urls"`
FailedURLs int `json:"failed_urls"`
ConfigID *uint `json:"config_id"`
ProgressData map[string]interface{} `json:"progress_data"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
StartedAt *time.Time `json:"started_at"`
CompletedAt *time.Time `json:"completed_at"`
}
// GoogleIndexTaskItemInput Google索引任务项输入
type GoogleIndexTaskItemInput struct {
TaskID uint `json:"task_id" binding:"required"`
URL string `json:"url" binding:"required,url"`
}
// GoogleIndexTaskItemOutput Google索引任务项输出
type GoogleIndexTaskItemOutput struct {
ID uint `json:"id"`
TaskID uint `json:"task_id"`
URL string `json:"url"`
Status string `json:"status"`
IndexStatus string `json:"index_status"`
ErrorMessage string `json:"error_message"`
InspectResult string `json:"inspect_result"`
MobileFriendly bool `json:"mobile_friendly"`
LastCrawled *time.Time `json:"last_crawled"`
StatusCode int `json:"status_code"`
StartedAt *time.Time `json:"started_at"`
CompletedAt *time.Time `json:"completed_at"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
// GoogleIndexURLStatusInput Google索引URL状态输入
type GoogleIndexURLStatusInput struct {
URL string `json:"url" binding:"required,url"`
IndexStatus string `json:"index_status" binding:"required"`
}
// GoogleIndexURLStatusOutput Google索引URL状态输出
type GoogleIndexURLStatusOutput struct {
ID uint `json:"id"`
URL string `json:"url"`
IndexStatus string `json:"index_status"`
LastChecked time.Time `json:"last_checked"`
CanonicalURL *string `json:"canonical_url"`
LastCrawled *time.Time `json:"last_crawled"`
ChangeFreq *string `json:"change_freq"`
Priority *float64 `json:"priority"`
MobileFriendly bool `json:"mobile_friendly"`
RobotsBlocked bool `json:"robots_blocked"`
LastError *string `json:"last_error"`
StatusCode int `json:"status_code"`
StatusCodeText string `json:"status_code_text"`
CheckCount int `json:"check_count"`
SuccessCount int `json:"success_count"`
FailureCount int `json:"failure_count"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
// GoogleIndexBatchRequest 批量处理请求
type GoogleIndexBatchRequest struct {
URLs []string `json:"urls" binding:"required,min=1,max=1000"`
Operation string `json:"operation" binding:"required,oneof=check_index submit_sitemap ping"` // 操作类型
}
// GoogleIndexBatchResponse 批量处理响应
type GoogleIndexBatchResponse struct {
Success bool `json:"success"`
Results []string `json:"results,omitempty"`
Message string `json:"message,omitempty"`
Total int `json:"total"`
Processed int `json:"processed"`
Failed int `json:"failed"`
}
// GoogleIndexStatusResponse 索引状态响应
type GoogleIndexStatusResponse struct {
Enabled bool `json:"enabled"`
SiteURL string `json:"site_url"`
LastCheckTime time.Time `json:"last_check_time"`
TotalURLs int `json:"total_urls"`
IndexedURLs int `json:"indexed_urls"`
NotIndexedURLs int `json:"not_indexed_urls"`
ErrorURLs int `json:"error_urls"`
LastSitemapSubmit time.Time `json:"last_sitemap_submit"`
AuthValid bool `json:"auth_valid"`
}
// GoogleIndexTaskListResponse 任务列表响应
type GoogleIndexTaskListResponse struct {
Tasks []GoogleIndexTaskOutput `json:"tasks"`
Total int64 `json:"total"`
Page int `json:"page"`
PageSize int `json:"page_size"`
TotalPages int `json:"total_pages"`
}
// GoogleIndexTaskItemPageResponse 任务项分页响应
type GoogleIndexTaskItemPageResponse struct {
Items []GoogleIndexTaskItemOutput `json:"items"`
Total int64 `json:"total"`
Page int `json:"page"`
Size int `json:"size"`
}

29
db/dto/request.go
View File

@@ -132,3 +132,32 @@ type SearchRequest struct {
Page int `json:"page"`
Limit int `json:"limit"`
}
// CreateTaskRequest 创建任务请求
type CreateTaskRequest struct {
Name string `json:"name" binding:"required"`
Description string `json:"description"`
TaskType string `json:"task_type" binding:"required"`
ConfigID *uint `json:"config_id"`
}
// CreateTaskItemRequest 创建任务项请求
type CreateTaskItemRequest struct {
URL string `json:"url"`
InputData map[string]interface{} `json:"input_data"`
}
// QueryTaskRequest 查询任务请求
type QueryTaskRequest struct {
Page int `json:"page" form:"page"`
PageSize int `json:"page_size" form:"page_size"`
Status string `json:"status" form:"status"`
Type string `json:"type" form:"type"`
}
// QueryTaskItemRequest 查询任务项请求
type QueryTaskItemRequest struct {
Page int `json:"page" form:"page"`
PageSize int `json:"page_size" form:"page_size"`
Status string `json:"status" form:"status"`
}

28
db/entity/google_index_config_constants.go Normal file
View File

@@ -0,0 +1,28 @@
package entity
// GoogleIndexConfigKeys Google索引配置键常量
const (
// 通用配置
GoogleIndexConfigKeyEnabled = "google_index_enabled" // 是否启用Google索引功能
GoogleIndexConfigKeySiteURL = "google_index_site_url" // 网站URL
GoogleIndexConfigKeySiteName = "google_index_site_name" // 网站名称
// 认证配置
GoogleIndexConfigKeyCredentialsFile = "google_index_credentials_file" // 凭证文件路径
GoogleIndexConfigKeyClientEmail = "google_index_client_email" // 客户端邮箱
GoogleIndexConfigKeyClientID = "google_index_client_id" // 客户端ID
GoogleIndexConfigKeyPrivateKey = "google_index_private_key" // 私钥（加密存储）
GoogleIndexConfigKeyToken = "google_index_token" // 访问令牌
// 调度配置
GoogleIndexConfigKeyCheckInterval = "google_index_check_interval" // 检查间隔（分钟）
GoogleIndexConfigKeyBatchSize = "google_index_batch_size" // 批处理大小
GoogleIndexConfigKeyConcurrency = "google_index_concurrency" // 并发数
GoogleIndexConfigKeyRetryAttempts = "google_index_retry_attempts" // 重试次数
GoogleIndexConfigKeyRetryDelay = "google_index_retry_delay" // 重试延迟（秒）
// 网站地图配置
GoogleIndexConfigKeyAutoSitemap = "google_index_auto_sitemap" // 自动提交网站地图
GoogleIndexConfigKeySitemapPath = "google_index_sitemap_path" // 网站地图路径
GoogleIndexConfigKeySitemapSchedule = "google_index_sitemap_schedule" // 网站地图调度
)

21
db/entity/task.go
View File

@@ -24,21 +24,24 @@ type TaskType string
const (
TaskTypeBatchTransfer TaskType = "batch_transfer" // 批量转存
TaskTypeExpansion TaskType = "expansion" // 账号扩容
TaskTypeGoogleIndex TaskType = "google_index" // Google索引
)
// Task 任务表
type Task struct {
ID uint `json:"id" gorm:"primaryKey;autoIncrement"`
Title string `json:"title" gorm:"size:255;not null;comment:任务标题"`
Name string `json:"name" gorm:"size:255;not null;default:'';comment:任务名称"`
Type TaskType `json:"type" gorm:"size:50;not null;comment:任务类型"`
Status TaskStatus `json:"status" gorm:"size:20;not null;default:pending;comment:任务状态"`
Description string `json:"description" gorm:"type:text;comment:任务描述"`
// 进度信息
TotalItems int `json:"total_items" gorm:"not null;default:0;comment:总项目数"`
ProcessedItems int `json:"processed_items" gorm:"not null;default:0;comment:已处理项目数"`
SuccessItems int `json:"success_items" gorm:"not null;default:0;comment:成功项目数"`
FailedItems int `json:"failed_items" gorm:"not null;default:0;comment:失败项目数"`
TotalItems int `json:"total_items" gorm:"not null;default:0;comment:总项目数"`
ProcessedItems int `json:"processed_items" gorm:"not null;default:0;comment:已处理项目数"`
SuccessItems int `json:"success_items" gorm:"not null;default:0;comment:成功项目数"`
FailedItems int `json:"failed_items" gorm:"not null;default:0;comment:失败项目数"`
Progress float64 `json:"progress" gorm:"not null;default:0.0;comment:任务进度"`
// 任务配置 (JSON格式存储)
Config string `json:"config" gorm:"type:text;comment:任务配置"`
@@ -46,6 +49,16 @@ type Task struct {
// 任务消息
Message string `json:"message" gorm:"type:text;comment:任务消息"`
// 进度数据 (JSON格式存储)
ProgressData string `json:"progress_data" gorm:"type:text;comment:进度数据"`
// Google索引特有字段 (当Type为google_index时使用)
IndexedURLs int `json:"indexed_urls" gorm:"default:0;comment:已索引URL数量"`
FailedURLs int `json:"failed_urls" gorm:"default:0;comment:失败URL数量"`
// 配置关联 (用于Google索引任务)
ConfigID *uint `json:"config_id" gorm:"comment:配置ID"`
// 时间信息
StartedAt *time.Time `json:"started_at" gorm:"comment:开始时间"`
CompletedAt *time.Time `json:"completed_at" gorm:"comment:完成时间"`

8
db/entity/task_item.go
View File

@@ -35,6 +35,14 @@ type TaskItem struct {
// 处理日志 (可选，用于记录详细的处理过程)
ProcessLog string `json:"process_log" gorm:"type:text;comment:处理日志"`
// Google索引特有字段 (当任务类型为google_index时使用)
URL string `json:"url" gorm:"size:2048;comment:URL (Google索引专用)"`
IndexStatus string `json:"index_status" gorm:"size:50;comment:索引状态 (Google索引专用)"`
InspectResult string `json:"inspect_result" gorm:"type:text;comment:检查结果 (Google索引专用)"`
MobileFriendly bool `json:"mobile_friendly" gorm:"default:false;comment:是否移动友好 (Google索引专用)"`
LastCrawled *time.Time `json:"last_crawled" gorm:"comment:最后抓取时间 (Google索引专用)"`
StatusCode int `json:"status_code" gorm:"default:0;comment:HTTP状态码 (Google索引专用)"`
// 时间信息
ProcessedAt *time.Time `json:"processed_at" gorm:"comment:处理时间"`
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`

28
db/repo/task_item_repository.go
View File

@@ -12,6 +12,7 @@ import (
type TaskItemRepository interface {
GetByID(id uint) (*entity.TaskItem, error)
Create(item *entity.TaskItem) error
Update(item *entity.TaskItem) error
Delete(id uint) error
DeleteByTaskID(taskID uint) error
GetByTaskIDAndStatus(taskID uint, status string) ([]*entity.TaskItem, error)
@@ -49,6 +50,33 @@ func (r *TaskItemRepositoryImpl) Create(item *entity.TaskItem) error {
return r.db.Create(item).Error
}
// Update 更新任务项
func (r *TaskItemRepositoryImpl) Update(item *entity.TaskItem) error {
startTime := utils.GetCurrentTime()
err := r.db.Model(&entity.TaskItem{}).Where("id = ?", item.ID).Updates(map[string]interface{}{
"status": item.Status,
"error_message": item.ErrorMessage,
"index_status": item.IndexStatus,
"mobile_friendly": item.MobileFriendly,
"last_crawled": item.LastCrawled,
"status_code": item.StatusCode,
"input_data": item.InputData,
"output_data": item.OutputData,
"process_log": item.ProcessLog,
"url": item.URL,
"inspect_result": item.InspectResult,
"processed_at": item.ProcessedAt,
"updated_at": time.Now(),
}).Error
updateDuration := time.Since(startTime)
if err != nil {
utils.Error("Update任务项失败: ID=%d, 错误=%v, 更新耗时=%v", item.ID, err, updateDuration)
return err
}
utils.Debug("Update任务项成功: ID=%d, 更新耗时=%v", item.ID, updateDuration)
return nil
}
// Delete 删除任务项
func (r *TaskItemRepositoryImpl) Delete(id uint) error {
return r.db.Delete(&entity.TaskItem{}, id).Error

20
go.mod
View File

@@ -15,19 +15,32 @@ require (
github.com/robfig/cron/v3 v3.0.1
github.com/silenceper/wechat/v2 v2.1.10
golang.org/x/crypto v0.41.0
golang.org/x/oauth2 v0.30.0
google.golang.org/api v0.197.0
gorm.io/driver/postgres v1.6.0
gorm.io/gorm v1.30.0
)
require (
cloud.google.com/go/auth v0.9.3 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect
cloud.google.com/go/compute/metadata v0.5.0 // indirect
github.com/andybalholm/brotli v1.1.1 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bradfitz/gomemcache v0.0.0-20220106215444-fb4bf637b56d // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-redis/redis/v8 v8.11.5 // indirect
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/google/s2a-go v0.1.8 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
github.com/googleapis/gax-go/v2 v2.13.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.66.1 // indirect
@@ -36,8 +49,15 @@ require (
github.com/tidwall/gjson v1.14.1 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect
go.opentelemetry.io/otel v1.29.0 // indirect
go.opentelemetry.io/otel/metric v1.29.0 // indirect
go.opentelemetry.io/otel/trace v1.29.0 // indirect
go.yaml.in/yaml/v2 v2.4.2 // indirect
golang.org/x/image v0.32.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect
google.golang.org/grpc v1.66.1 // indirect
)
require (

95
go.sum
View File

@@ -1,3 +1,11 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go/auth v0.9.3 h1:VOEUIAADkkLtyfr3BLa3R8Ed/j6w1jTBmARx+wb5w5U=
cloud.google.com/go/auth v0.9.3/go.mod h1:7z6VY+7h3KUdRov5F1i8NDP5ZzWKYmEPO842BgCsmTk=
cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY=
cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc=
cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY=
cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk=
github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
github.com/alicebob/miniredis/v2 v2.30.0 h1:uA3uhDbCxfO9+DI/DuGeAMr9qI+noVWwGPNTFuKID5M=
@@ -13,22 +21,31 @@ github.com/bytedance/sonic v1.13.3/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1
github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA=
github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4=
github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/fogleman/gg v1.3.0 h1:/7zJX8F6AaYQc57WQCyN9cAIz+4bCJGO9B+dyW29am8=
github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
@@ -44,6 +61,11 @@ github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fq
github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk=
github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ=
github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
@@ -72,23 +94,43 @@ github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1
github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM=
github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw=
github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA=
github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s=
github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A=
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
@@ -166,6 +208,7 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
@@ -213,6 +256,16 @@ github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3i
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64 h1:5mLPGnFdSsevFRFc9q3yYbBkB6tsm4aCwwQV/j1JQAQ=
github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8=
go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw=
go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8=
go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc=
go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8=
go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4=
go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
@@ -226,24 +279,38 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4=
golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/image v0.32.0 h1:6lZQWq75h7L5IWNk0r+SCpUJ6tUVd3v4ZHnbRKLkUDQ=
golang.org/x/image v0.32.0/go.mod h1:/R37rrQmKXtO6tYXAjtDLwQgFLHmhW+V6ayXlxzP2Pc=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE=
golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI=
golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -274,18 +341,44 @@ golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=
golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U=
golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/api v0.197.0 h1:x6CwqQLsFiA5JKAiGyGBjc2bNtHtLddhJCE2IKuhhcQ=
google.golang.org/api v0.197.0/go.mod h1:AuOuo20GoQ331nq7DquGHlU6d+2wN2fZ8O0ta60nRNw=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 h1:BulPr26Jqjnd4eYDVe+YvyR7Yc2vJGkO5/0UxD0/jZU=
google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d h1:kHjw/5UfflP/L5EbledDrcG4C2597RtymmGRZvHiCuY=
google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d/go.mod h1:mw8MG/Qz5wfgYr6VqVCiZcHe/GJEfI+oGGDCohaVgB0=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
google.golang.org/grpc v1.66.1 h1:hO5qAXR19+/Z44hmvIM4dQFMSYX9XcWsByfoxutBpAM=
google.golang.org/grpc v1.66.1/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
@@ -314,4 +407,6 @@ gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4=
gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo=
gorm.io/gorm v1.30.0 h1:qbT5aPv1UH8gI99OsRlvDToLxW5zR7FzS9acZDOZcgs=
gorm.io/gorm v1.30.0/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=

846
handlers/google_index_handler.go Normal file
View File

@@ -0,0 +1,846 @@
package handlers
import (
"encoding/json"
"fmt"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
"github.com/ctwj/urldb/db/converter"
"github.com/ctwj/urldb/db/dto"
"github.com/ctwj/urldb/db/entity"
"github.com/ctwj/urldb/db/repo"
"github.com/ctwj/urldb/pkg/google" // 添加google包导入
"github.com/ctwj/urldb/task"
"github.com/ctwj/urldb/utils"
"github.com/gin-gonic/gin"
)
// GoogleIndexHandler Google索引处理程序
type GoogleIndexHandler struct {
repoMgr *repo.RepositoryManager
taskManager *task.TaskManager
}
// NewGoogleIndexHandler 创建Google索引处理程序
func NewGoogleIndexHandler(
repoMgr *repo.RepositoryManager,
taskManager *task.TaskManager,
) *GoogleIndexHandler {
return &GoogleIndexHandler{
repoMgr: repoMgr,
taskManager: taskManager,
}
}
// GetConfig 获取Google索引配置
func (h *GoogleIndexHandler) GetConfig(c *gin.Context) {
// 获取通用配置
enabledStr, err := h.repoMgr.SystemConfigRepository.GetConfigValue(entity.GoogleIndexConfigKeyEnabled)
if err != nil {
enabledStr = "false"
}
enabled := enabledStr == "true" || enabledStr == "1"
siteURL, err := h.repoMgr.SystemConfigRepository.GetConfigValue(entity.GoogleIndexConfigKeySiteURL)
if err != nil {
siteURL = ""
}
siteName, err := h.repoMgr.SystemConfigRepository.GetConfigValue(entity.GoogleIndexConfigKeySiteName)
if err != nil {
siteName = ""
}
// 获取调度配置
checkIntervalStr, err := h.repoMgr.SystemConfigRepository.GetConfigValue(entity.GoogleIndexConfigKeyCheckInterval)
if err != nil {
checkIntervalStr = "60"
}
checkInterval, _ := strconv.Atoi(checkIntervalStr)
batchSizeStr, err := h.repoMgr.SystemConfigRepository.GetConfigValue(entity.GoogleIndexConfigKeyBatchSize)
if err != nil {
batchSizeStr = "100"
}
batchSize, _ := strconv.Atoi(batchSizeStr)
concurrencyStr, err := h.repoMgr.SystemConfigRepository.GetConfigValue(entity.GoogleIndexConfigKeyConcurrency)
if err != nil {
concurrencyStr = "5"
}
concurrency, _ := strconv.Atoi(concurrencyStr)
retryAttemptsStr, err := h.repoMgr.SystemConfigRepository.GetConfigValue(entity.GoogleIndexConfigKeyRetryAttempts)
if err != nil {
retryAttemptsStr = "3"
}
retryAttempts, _ := strconv.Atoi(retryAttemptsStr)
retryDelayStr, err := h.repoMgr.SystemConfigRepository.GetConfigValue(entity.GoogleIndexConfigKeyRetryDelay)
if err != nil {
retryDelayStr = "2"
}
retryDelay, _ := strconv.Atoi(retryDelayStr)
// 获取网站地图配置
autoSitemapStr, err := h.repoMgr.SystemConfigRepository.GetConfigValue(entity.GoogleIndexConfigKeyAutoSitemap)
if err != nil {
autoSitemapStr = "false"
}
autoSitemap := autoSitemapStr == "true" || autoSitemapStr == "1"
sitemapPath, err := h.repoMgr.SystemConfigRepository.GetConfigValue(entity.GoogleIndexConfigKeySitemapPath)
if err != nil {
sitemapPath = "/sitemap.xml"
}
sitemapSchedule, err := h.repoMgr.SystemConfigRepository.GetConfigValue(entity.GoogleIndexConfigKeySitemapSchedule)
if err != nil {
sitemapSchedule = "@daily"
}
config := dto.GoogleIndexConfigGeneral{
Enabled: enabled,
SiteURL: siteURL,
SiteName: siteName,
}
scheduleConfig := dto.GoogleIndexConfigSchedule{
CheckInterval: checkInterval,
BatchSize: batchSize,
Concurrency: concurrency,
RetryAttempts: retryAttempts,
RetryDelay: retryDelay,
}
sitemapConfig := dto.GoogleIndexConfigSitemap{
AutoSitemap: autoSitemap,
SitemapPath: sitemapPath,
SitemapSchedule: sitemapSchedule,
}
result := gin.H{
"general": config,
"schedule": scheduleConfig,
"sitemap": sitemapConfig,
"is_running": false, // 不再有独立的调度器，使用统一任务管理器
}
SuccessResponse(c, result)
}
// UpdateConfig 更新Google索引配置
func (h *GoogleIndexHandler) UpdateConfig(c *gin.Context) {
var req struct {
General dto.GoogleIndexConfigGeneral `json:"general"`
Schedule dto.GoogleIndexConfigSchedule `json:"schedule"`
Sitemap dto.GoogleIndexConfigSitemap `json:"sitemap"`
}
if err := c.ShouldBindJSON(&req); err != nil {
ErrorResponse(c, "参数错误: "+err.Error(), http.StatusBadRequest)
return
}
username, _ := c.Get("username")
clientIP, _ := c.Get("client_ip")
utils.Info("GoogleIndexHandler.UpdateConfig - 用户更新Google索引配置 - 用户: %s, IP: %s", username, clientIP)
// 准备要更新的配置项
configs := []entity.SystemConfig{
{
Key: entity.GoogleIndexConfigKeyEnabled,
Value: strconv.FormatBool(req.General.Enabled),
Type: entity.ConfigTypeBool,
},
{
Key: entity.GoogleIndexConfigKeySiteURL,
Value: req.General.SiteURL,
Type: entity.ConfigTypeString,
},
{
Key: entity.GoogleIndexConfigKeySiteName,
Value: req.General.SiteName,
Type: entity.ConfigTypeString,
},
{
Key: entity.GoogleIndexConfigKeyCheckInterval,
Value: strconv.Itoa(req.Schedule.CheckInterval),
Type: entity.ConfigTypeInt,
},
{
Key: entity.GoogleIndexConfigKeyBatchSize,
Value: strconv.Itoa(req.Schedule.BatchSize),
Type: entity.ConfigTypeInt,
},
{
Key: entity.GoogleIndexConfigKeyConcurrency,
Value: strconv.Itoa(req.Schedule.Concurrency),
Type: entity.ConfigTypeInt,
},
{
Key: entity.GoogleIndexConfigKeyRetryAttempts,
Value: strconv.Itoa(req.Schedule.RetryAttempts),
Type: entity.ConfigTypeInt,
},
{
Key: entity.GoogleIndexConfigKeyRetryDelay,
Value: strconv.Itoa(req.Schedule.RetryDelay),
Type: entity.ConfigTypeInt,
},
{
Key: entity.GoogleIndexConfigKeyAutoSitemap,
Value: strconv.FormatBool(req.Sitemap.AutoSitemap),
Type: entity.ConfigTypeBool,
},
{
Key: entity.GoogleIndexConfigKeySitemapPath,
Value: req.Sitemap.SitemapPath,
Type: entity.ConfigTypeString,
},
{
Key: entity.GoogleIndexConfigKeySitemapSchedule,
Value: req.Sitemap.SitemapSchedule,
Type: entity.ConfigTypeString,
},
}
// 批量更新配置
err := h.repoMgr.SystemConfigRepository.UpsertConfigs(configs)
if err != nil {
utils.Error("更新系统配置失败: %v", err)
ErrorResponse(c, "更新配置失败", http.StatusInternalServerError)
return
}
utils.Info("Google索引配置更新成功 - 用户: %s, IP: %s", username, clientIP)
SuccessResponse(c, gin.H{
"message": "配置更新成功",
})
}
// CreateTask 创建Google索引任务
func (h *GoogleIndexHandler) CreateTask(c *gin.Context) {
var req dto.GoogleIndexTaskInput
if err := c.ShouldBindJSON(&req); err != nil {
ErrorResponse(c, "参数错误: "+err.Error(), http.StatusBadRequest)
return
}
username, _ := c.Get("username")
clientIP, _ := c.Get("client_ip")
utils.Info("GoogleIndexHandler.CreateTask - 用户创建Google索引任务 - 用户: %s, 任务类型: %s, 任务标题: %s, IP: %s", username, req.Type, req.Title, clientIP)
// 创建通用任务
task, err := h.taskManager.CreateTask(string(entity.TaskTypeGoogleIndex), req.Title, req.Description, req.ConfigID)
if err != nil {
utils.Error("创建Google索引任务失败: %v", err)
ErrorResponse(c, "创建任务失败", http.StatusInternalServerError)
return
}
// 根据任务类型创建任务项
var taskItems []*entity.TaskItem
switch req.Type {
case "url_indexing", "status_check", "batch_index":
// 为每个URL创建任务项
for _, url := range req.URLs {
itemData := map[string]interface{}{
"urls": []string{url},
"operation": req.Type,
}
itemDataJSON, _ := json.Marshal(itemData)
taskItem := &entity.TaskItem{
URL: url,
InputData: string(itemDataJSON),
}
taskItems = append(taskItems, taskItem)
}
case "sitemap_submit":
// 为网站地图创建任务项
itemData := map[string]interface{}{
"sitemap_url": req.SitemapURL,
"operation": "sitemap_submit",
}
itemDataJSON, _ := json.Marshal(itemData)
taskItem := &entity.TaskItem{
URL: req.SitemapURL,
InputData: string(itemDataJSON),
}
taskItems = append(taskItems, taskItem)
}
// 批量创建任务项
err = h.taskManager.CreateTaskItems(task.ID, taskItems)
if err != nil {
utils.Error("创建任务项失败: %v", err)
ErrorResponse(c, "创建任务项失败", http.StatusInternalServerError)
return
}
utils.Info("Google索引任务创建完成: %d, 任务类型: %s", task.ID, req.Type)
SuccessResponse(c, gin.H{
"task_id": task.ID,
"total_items": len(taskItems),
"message": "任务创建成功",
})
}
// StartTask 启动Google索引任务
func (h *GoogleIndexHandler) StartTask(c *gin.Context) {
taskIDStr := c.Param("id")
taskID, err := strconv.ParseUint(taskIDStr, 10, 32)
if err != nil {
ErrorResponse(c, "无效的任务ID", http.StatusBadRequest)
return
}
username, _ := c.Get("username")
clientIP, _ := c.Get("client_ip")
utils.Info("GoogleIndexHandler.StartTask - 用户启动Google索引任务 - 用户: %s, 任务ID: %d, IP: %s", username, taskID, clientIP)
// 使用任务管理器启动任务
err = h.taskManager.StartTask(uint(taskID))
if err != nil {
utils.Error("启动Google索引任务失败: %v", err)
ErrorResponse(c, "启动任务失败: "+err.Error(), http.StatusInternalServerError)
return
}
utils.Info("Google索引任务启动成功: %d", taskID)
SuccessResponse(c, gin.H{
"message": "任务启动成功",
})
}
// GetTaskStatus 获取任务状态
func (h *GoogleIndexHandler) GetTaskStatus(c *gin.Context) {
taskIDStr := c.Param("id")
taskID, err := strconv.ParseUint(taskIDStr, 10, 32)
if err != nil {
ErrorResponse(c, "无效的任务ID", http.StatusBadRequest)
return
}
task, err := h.taskManager.GetTask(uint(taskID))
if err != nil {
ErrorResponse(c, "获取任务失败", http.StatusInternalServerError)
return
}
if task == nil {
ErrorResponse(c, "任务不存在", http.StatusNotFound)
return
}
// 获取任务项统计
stats, err := h.taskManager.GetTaskItemStats(task.ID)
if err != nil {
utils.Error("获取任务项统计失败: %v", err)
stats = make(map[string]int)
}
taskOutput := converter.TaskToGoogleIndexTaskOutput(task, stats)
result := gin.H{
"id": taskOutput.ID,
"name": taskOutput.Name,
"type": taskOutput.Type,
"status": taskOutput.Status,
"description": taskOutput.Description,
"progress": taskOutput.Progress,
"total_items": taskOutput.TotalItems,
"processed_items": taskOutput.ProcessedItems,
"successful_items": taskOutput.SuccessfulItems,
"failed_items": taskOutput.FailedItems,
"pending_items": taskOutput.PendingItems,
"processing_items": taskOutput.ProcessingItems,
"indexed_urls": taskOutput.IndexedURLs,
"failed_urls": taskOutput.FailedURLs,
"started_at": taskOutput.StartedAt,
"completed_at": taskOutput.CompletedAt,
"created_at": taskOutput.CreatedAt,
"updated_at": taskOutput.UpdatedAt,
"progress_data": taskOutput.ProgressData,
"stats": stats,
}
SuccessResponse(c, result)
}
// GetTasks 获取任务列表
func (h *GoogleIndexHandler) GetTasks(c *gin.Context) {
pageStr := c.DefaultQuery("page", "1")
pageSizeStr := c.DefaultQuery("page_size", "10")
taskTypeStr := c.Query("type")
statusStr := c.Query("status")
page, _ := strconv.Atoi(pageStr)
if page < 1 {
page = 1
}
pageSize, _ := strconv.Atoi(pageSizeStr)
if pageSize < 1 || pageSize > 100 {
pageSize = 10
}
// 根据参数筛选任务类型，如果有指定则使用，否则默认为Google索引类型
taskType := string(entity.TaskTypeGoogleIndex)
if taskTypeStr != "" {
taskType = taskTypeStr
}
// 获取指定状态的任务，默认查找所有状态
status := statusStr
// 获取任务列表 - 目前我们没有Query方法，直接获取所有任务然后做筛选
tasksList, total, err := h.repoMgr.TaskRepository.GetList(page, pageSize, taskType, status)
if err != nil {
ErrorResponse(c, "获取任务列表失败", http.StatusInternalServerError)
return
}
taskOutputs := make([]dto.GoogleIndexTaskOutput, len(tasksList))
for i, task := range tasksList {
// 获取任务统计信息
stats, err := h.taskManager.GetTaskItemStats(task.ID)
if err != nil {
stats = make(map[string]int)
}
taskOutputs[i] = converter.TaskToGoogleIndexTaskOutput(task, stats)
}
result := dto.GoogleIndexTaskListResponse{
Tasks: taskOutputs,
Total: total,
Page: page,
PageSize: pageSize,
TotalPages: int((total + int64(pageSize) - 1) / int64(pageSize)),
}
SuccessResponse(c, result)
}
// GetTaskItems 获取任务项列表
func (h *GoogleIndexHandler) GetTaskItems(c *gin.Context) {
taskIDStr := c.Param("id")
taskID, err := strconv.ParseUint(taskIDStr, 10, 32)
if err != nil {
ErrorResponse(c, "无效的任务ID", http.StatusBadRequest)
return
}
pageStr := c.DefaultQuery("page", "1")
pageSizeStr := c.DefaultQuery("page_size", "50")
statusStr := c.Query("status")
page, _ := strconv.Atoi(pageStr)
if page < 1 {
page = 1
}
pageSize, _ := strconv.Atoi(pageSizeStr)
if pageSize < 1 || pageSize > 1000 {
pageSize = 50
}
// 获取任务项列表
items, total, err := h.taskManager.QueryTaskItems(uint(taskID), page, pageSize, statusStr)
if err != nil {
ErrorResponse(c, "获取任务项列表失败", http.StatusInternalServerError)
return
}
// 注意：我们还没有TaskItemToGoogleIndexTaskItemOutput转换器，需要创建一个
itemOutputs := make([]dto.GoogleIndexTaskItemOutput, len(items))
for i, item := range items {
// 手动构建输出结构
itemOutputs[i] = dto.GoogleIndexTaskItemOutput{
ID: item.ID,
TaskID: item.TaskID,
URL: item.URL,
Status: string(item.Status),
IndexStatus: item.IndexStatus,
ErrorMessage: item.ErrorMessage,
InspectResult: item.InspectResult,
MobileFriendly: item.MobileFriendly,
LastCrawled: item.LastCrawled,
StatusCode: item.StatusCode,
CreatedAt: item.CreatedAt,
UpdatedAt: item.UpdatedAt,
StartedAt: item.ProcessedAt, // 任务项处理完成时间
CompletedAt: item.ProcessedAt,
}
}
result := dto.GoogleIndexTaskItemPageResponse{
Items: itemOutputs,
Total: total,
Page: page,
Size: pageSize,
}
SuccessResponse(c, result)
}
// UploadCredentials 上传Google索引凭据
func (h *GoogleIndexHandler) UploadCredentials(c *gin.Context) {
// 获取上传的文件
file, err := c.FormFile("file")
if err != nil {
ErrorResponse(c, "未提供凭据文件", http.StatusBadRequest)
return
}
// 验证文件扩展名必须是.json
ext := strings.ToLower(filepath.Ext(file.Filename))
if ext != ".json" {
ErrorResponse(c, "仅支持JSON格式的凭据文件", http.StatusBadRequest)
return
}
// 验证文件大小（限制5MB）
if file.Size > 5*1024*1024 {
ErrorResponse(c, "文件大小不能超过5MB", http.StatusBadRequest)
return
}
// 确保data目录存在
dataDir := "./data"
if err := os.MkdirAll(dataDir, 0755); err != nil {
ErrorResponse(c, "创建数据目录失败", http.StatusInternalServerError)
return
}
// 使用固定的文件名保存凭据
fixedFileName := "google_credentials.json"
filePath := filepath.Join(dataDir, fixedFileName)
// 保存文件
if err := c.SaveUploadedFile(file, filePath); err != nil {
ErrorResponse(c, "保存凭据文件失败", http.StatusInternalServerError)
return
}
// 设置文件权限
if err := os.Chmod(filePath, 0600); err != nil {
utils.Warn("设置凭据文件权限失败: %v", err)
}
// 返回成功响应
accessPath := filepath.Join("data", fixedFileName)
response := map[string]interface{}{
"success": true,
"message": "凭据文件上传成功",
"file_name": fixedFileName,
"file_path": accessPath,
"full_path": filePath,
}
SuccessResponse(c, response)
}
// makeSafeFileName 生成安全的文件名，移除危险字符
func (h *GoogleIndexHandler) makeSafeFileName(filename string) string {
// 移除路径分隔符和特殊字符
safeName := strings.ReplaceAll(filename, "/", "_")
safeName = strings.ReplaceAll(safeName, "\\", "_")
safeName = strings.ReplaceAll(safeName, "..", "_")
// 限制文件名长度
if len(safeName) > 100 {
ext := filepath.Ext(safeName)
name := safeName[:100-len(ext)]
safeName = name + ext
}
return safeName
}
// ValidateCredentials 验证Google索引凭据
func (h *GoogleIndexHandler) ValidateCredentials(c *gin.Context) {
var req struct {
CredentialsFile string `json:"credentials_file" binding:"required"`
}
if err := c.ShouldBindJSON(&req); err != nil {
ErrorResponse(c, "请求参数错误: "+err.Error(), http.StatusBadRequest)
return
}
// 检查凭据文件是否存在
if _, err := os.Stat(req.CredentialsFile); os.IsNotExist(err) {
ErrorResponse(c, "凭据文件不存在", http.StatusBadRequest)
return
}
// 尝试创建Google客户端并验证凭据
config, err := h.loadCredentials(req.CredentialsFile)
if err != nil {
ErrorResponse(c, "凭据格式错误: "+err.Error(), http.StatusBadRequest)
return
}
// 验证凭据是否有效（尝试获取token）
err = h.getValidToken(config)
if err != nil {
ErrorResponse(c, "凭据验证失败: "+err.Error(), http.StatusBadRequest)
return
}
response := map[string]interface{}{
"success": true,
"message": "凭据验证成功",
"valid": true,
}
SuccessResponse(c, response)
}
// loadCredentials 从文件加载凭据
func (h *GoogleIndexHandler) loadCredentials(credentialsFile string) (*google.Config, error) {
// 从pkg/google/client.go导入的Config
// 注意：我们需要一个方法来安全地加载凭据
// 为了简化，我们只是检查文件是否可以读取以及格式是否正确
data, err := os.ReadFile(credentialsFile)
if err != nil {
return nil, fmt.Errorf("无法读取凭据文件: %v", err)
}
// 验证是否为有效的JSON
var temp map[string]interface{}
if err := json.Unmarshal(data, &temp); err != nil {
return nil, fmt.Errorf("凭据文件格式不是有效的JSON: %v", err)
}
// 检查必需字段
requiredFields := []string{"type", "project_id", "private_key_id", "private_key", "client_email", "client_id"}
for _, field := range requiredFields {
if _, exists := temp[field]; !exists {
return nil, fmt.Errorf("凭据文件缺少必需字段: %s", field)
}
}
// 返回配置（简化处理，实际实现可能需要更复杂的逻辑）
return &google.Config{
CredentialsFile: credentialsFile,
}, nil
}
// getValidToken 获取有效的token
// GetConfigByKey 根据键获取Google索引配置
func (h *GoogleIndexHandler) GetConfigByKey(c *gin.Context) {
// 从URL参数获取配置键
key := c.Param("key")
if key == "" {
ErrorResponse(c, "配置键不能为空", http.StatusBadRequest)
return
}
username, _ := c.Get("username")
clientIP, _ := c.Get("client_ip")
utils.Info("GoogleIndexHandler.GetConfigByKey - 获取Google索引配置 - 用户: %s, 键: %s, IP: %s", username, key, clientIP)
// 根据键查找配置
config, err := h.repoMgr.SystemConfigRepository.FindByKey(key)
if err != nil {
// 如果配置不存在，返回默认值或空值
SuccessResponse(c, map[string]interface{}{
"group": "verification",
"key": key,
"value": "",
"type": "string",
})
return
}
// 返回配置项
SuccessResponse(c, map[string]interface{}{
"group": "verification",
"key": config.Key,
"value": config.Value,
"type": config.Type,
})
}
// UpdateGoogleIndexConfig 更新Google索引配置（支持分组配置）
func (h *GoogleIndexHandler) UpdateGoogleIndexConfig(c *gin.Context) {
var req dto.GoogleIndexConfigInput
if err := c.ShouldBindJSON(&req); err != nil {
ErrorResponse(c, "参数错误: "+err.Error(), http.StatusBadRequest)
return
}
username, _ := c.Get("username")
clientIP, _ := c.Get("client_ip")
utils.Info("GoogleIndexHandler.UpdateGoogleIndexConfig - 用户更新Google索引分组配置 - 用户: %s, 组: %s, 键: %s, IP: %s", username, req.Group, req.Key, clientIP)
// 处理不同的配置组
switch req.Group {
case "general":
switch req.Key {
case "general":
// 解析general配置
var generalConfig dto.GoogleIndexConfigGeneral
if err := json.Unmarshal([]byte(req.Value), &generalConfig); err != nil {
ErrorResponse(c, "通用配置格式错误: "+err.Error(), http.StatusBadRequest)
return
}
// 存储各个字段
generalConfigs := []entity.SystemConfig{
{Key: entity.GoogleIndexConfigKeyEnabled, Value: strconv.FormatBool(generalConfig.Enabled), Type: entity.ConfigTypeBool},
{Key: entity.GoogleIndexConfigKeySiteURL, Value: generalConfig.SiteURL, Type: entity.ConfigTypeString},
{Key: entity.GoogleIndexConfigKeySiteName, Value: generalConfig.SiteName, Type: entity.ConfigTypeString},
}
err := h.repoMgr.SystemConfigRepository.UpsertConfigs(generalConfigs)
if err != nil {
ErrorResponse(c, "保存通用配置失败: "+err.Error(), http.StatusInternalServerError)
return
}
default:
ErrorResponse(c, "未知的通用配置键: "+req.Key, http.StatusBadRequest)
return
}
case "auth":
switch req.Key {
case "credentials_file":
// 解析认证配置
var authConfig dto.GoogleIndexConfigAuth
if err := json.Unmarshal([]byte(req.Value), &authConfig); err != nil {
ErrorResponse(c, "认证配置格式错误: "+err.Error(), http.StatusBadRequest)
return
}
// 存储认证相关配置
authConfigs := []entity.SystemConfig{
{Key: entity.GoogleIndexConfigKeyCredentialsFile, Value: authConfig.CredentialsFile, Type: entity.ConfigTypeString},
{Key: entity.GoogleIndexConfigKeyClientEmail, Value: authConfig.ClientEmail, Type: entity.ConfigTypeString},
{Key: entity.GoogleIndexConfigKeyClientID, Value: authConfig.ClientID, Type: entity.ConfigTypeString},
{Key: entity.GoogleIndexConfigKeyPrivateKey, Value: authConfig.PrivateKey, Type: entity.ConfigTypeString},
}
err := h.repoMgr.SystemConfigRepository.UpsertConfigs(authConfigs)
if err != nil {
ErrorResponse(c, "保存认证配置失败: "+err.Error(), http.StatusInternalServerError)
return
}
default:
ErrorResponse(c, "未知的认证配置键: "+req.Key, http.StatusBadRequest)
return
}
case "schedule":
switch req.Key {
case "schedule":
// 解析调度配置
var scheduleConfig dto.GoogleIndexConfigSchedule
if err := json.Unmarshal([]byte(req.Value), &scheduleConfig); err != nil {
ErrorResponse(c, "调度配置格式错误: "+err.Error(), http.StatusBadRequest)
return
}
// 存储调度相关配置
scheduleConfigs := []entity.SystemConfig{
{Key: entity.GoogleIndexConfigKeyCheckInterval, Value: strconv.Itoa(scheduleConfig.CheckInterval), Type: entity.ConfigTypeInt},
{Key: entity.GoogleIndexConfigKeyBatchSize, Value: strconv.Itoa(scheduleConfig.BatchSize), Type: entity.ConfigTypeInt},
{Key: entity.GoogleIndexConfigKeyConcurrency, Value: strconv.Itoa(scheduleConfig.Concurrency), Type: entity.ConfigTypeInt},
{Key: entity.GoogleIndexConfigKeyRetryAttempts, Value: strconv.Itoa(scheduleConfig.RetryAttempts), Type: entity.ConfigTypeInt},
{Key: entity.GoogleIndexConfigKeyRetryDelay, Value: strconv.Itoa(scheduleConfig.RetryDelay), Type: entity.ConfigTypeInt},
}
err := h.repoMgr.SystemConfigRepository.UpsertConfigs(scheduleConfigs)
if err != nil {
ErrorResponse(c, "保存调度配置失败: "+err.Error(), http.StatusInternalServerError)
return
}
default:
ErrorResponse(c, "未知的调度配置键: "+req.Key, http.StatusBadRequest)
return
}
case "sitemap":
switch req.Key {
case "sitemap":
// 解析网站地图配置
var sitemapConfig dto.GoogleIndexConfigSitemap
if err := json.Unmarshal([]byte(req.Value), &sitemapConfig); err != nil {
ErrorResponse(c, "网站地图配置格式错误: "+err.Error(), http.StatusBadRequest)
return
}
// 存储网站地图相关配置
sitemapConfigs := []entity.SystemConfig{
{Key: entity.GoogleIndexConfigKeyAutoSitemap, Value: strconv.FormatBool(sitemapConfig.AutoSitemap), Type: entity.ConfigTypeBool},
{Key: entity.GoogleIndexConfigKeySitemapPath, Value: sitemapConfig.SitemapPath, Type: entity.ConfigTypeString},
{Key: entity.GoogleIndexConfigKeySitemapSchedule, Value: sitemapConfig.SitemapSchedule, Type: entity.ConfigTypeString},
}
err := h.repoMgr.SystemConfigRepository.UpsertConfigs(sitemapConfigs)
if err != nil {
ErrorResponse(c, "保存网站地图配置失败: "+err.Error(), http.StatusInternalServerError)
return
}
default:
ErrorResponse(c, "未知的网站地图配置键: "+req.Key, http.StatusBadRequest)
return
}
case "verification":
switch req.Key {
case "google_site_verification":
// 解析验证配置
var verificationConfig struct {
Code string `json:"code"`
}
if err := json.Unmarshal([]byte(req.Value), &verificationConfig); err != nil {
ErrorResponse(c, "验证配置格式错误: "+err.Error(), http.StatusBadRequest)
return
}
// 存储验证字符串
verificationConfigs := []entity.SystemConfig{
{Key: "google_site_verification_code", Value: verificationConfig.Code, Type: entity.ConfigTypeString},
}
err := h.repoMgr.SystemConfigRepository.UpsertConfigs(verificationConfigs)
if err != nil {
ErrorResponse(c, "保存验证配置失败: "+err.Error(), http.StatusInternalServerError)
return
}
default:
ErrorResponse(c, "未知的验证配置键: "+req.Key, http.StatusBadRequest)
return
}
default:
ErrorResponse(c, "未知的配置组: "+req.Group, http.StatusBadRequest)
return
}
utils.Info("Google索引分组配置更新成功 - 组: %s, 键: %s - 用户: %s, IP: %s", req.Group, req.Key, username, clientIP)
SuccessResponse(c, gin.H{
"message": "配置更新成功",
"group": req.Group,
"key": req.Key,
})
}
func (h *GoogleIndexHandler) getValidToken(config *google.Config) error {
// 这里应该使用Google的验证逻辑
// 为了简化我们返回一个模拟的验证过程
// 在实际实现中，应该使用Google API进行验证
// 尝试初始化Google客户端
client, err := google.NewClient(config)
if err != nil {
return fmt.Errorf("创建Google客户端失败: %v", err)
}
// 简单的验证：尝试获取网站列表，如果成功说明凭据有效
// 这里我们只检查客户端是否能成功初始化
// 在实际实现中，应该尝试执行一个API调用以验证凭据
_ = client // 使用client变量避免未使用警告
return nil
}

25
handlers/system_config_handler.go
View File

@@ -281,6 +281,31 @@ func GetPublicSystemConfig(c *gin.Context) {
SuccessResponse(c, configResponse)
}
// 新增：获取网站验证代码（公开访问）
func GetSiteVerificationCode(c *gin.Context) {
// 获取所有系统配置
configs, err := repoManager.SystemConfigRepository.GetOrCreateDefault()
if err != nil {
ErrorResponse(c, "获取系统配置失败", http.StatusInternalServerError)
return
}
// 转换为公共响应格式
configResponse := converter.SystemConfigToPublicResponse(configs)
// 只返回验证代码，确保安全性
verificationCode := ""
if verificationCodeVal, exists := configResponse["google_site_verification_code"]; exists {
if codeStr, ok := verificationCodeVal.(string); ok {
verificationCode = codeStr
}
}
SuccessResponse(c, gin.H{
"google_site_verification_code": verificationCode,
})
}
// 新增：配置监控端点
func GetConfigStatus(c *gin.Context) {
// 获取配置统计信息

30
main.go
View File

@@ -237,6 +237,17 @@ func main() {
reportHandler := handlers.NewReportHandler(repoManager.ReportRepository, repoManager.ResourceRepository)
copyrightClaimHandler := handlers.NewCopyrightClaimHandler(repoManager.CopyrightClaimRepository, repoManager.ResourceRepository)
// 创建Google索引任务处理器
googleIndexProcessor := task.NewGoogleIndexProcessor(repoManager)
// 创建Google索引处理器
googleIndexHandler := handlers.NewGoogleIndexHandler(repoManager, taskManager)
// 注册Google索引处理器到任务管理器
taskManager.RegisterProcessor(googleIndexProcessor)
utils.Info("Google索引功能已启用，注册到任务管理器")
// API路由
api := r.Group("/api")
{
@@ -363,6 +374,7 @@ func main() {
api.GET("/system/config/status", middleware.AuthMiddleware(), middleware.AdminMiddleware(), handlers.GetConfigStatus)
api.POST("/system/config/toggle-auto-process", middleware.AuthMiddleware(), middleware.AdminMiddleware(), handlers.ToggleAutoProcess)
api.GET("/public/system-config", handlers.GetPublicSystemConfig)
api.GET("/public/site-verification", handlers.GetSiteVerificationCode) // 网站验证代码（公开访问）
// 热播剧管理路由（查询接口无需认证）
api.GET("/hot-dramas", handlers.GetHotDramaList)
@@ -521,6 +533,21 @@ func main() {
api.POST("/sitemap/generate", middleware.AuthMiddleware(), middleware.AdminMiddleware(), handlers.GenerateSitemap)
api.GET("/sitemap/status", middleware.AuthMiddleware(), middleware.AdminMiddleware(), handlers.GetSitemapStatus)
api.POST("/sitemap/full-generate", middleware.AuthMiddleware(), middleware.AdminMiddleware(), handlers.GenerateFullSitemap)
// Google索引管理API
api.GET("/google-index/config", middleware.AuthMiddleware(), middleware.AdminMiddleware(), googleIndexHandler.GetConfig)
api.POST("/google-index/config", middleware.AuthMiddleware(), middleware.AdminMiddleware(), googleIndexHandler.UpdateConfig)
api.GET("/google-index/config/:key", middleware.AuthMiddleware(), middleware.AdminMiddleware(), googleIndexHandler.GetConfigByKey) // 根据键获取配置
api.POST("/google-index/config/update", middleware.AuthMiddleware(), middleware.AdminMiddleware(), googleIndexHandler.UpdateGoogleIndexConfig) // 分组配置更新
api.POST("/google-index/tasks", middleware.AuthMiddleware(), middleware.AdminMiddleware(), googleIndexHandler.CreateTask)
api.GET("/google-index/tasks", middleware.AuthMiddleware(), middleware.AdminMiddleware(), googleIndexHandler.GetTasks)
api.GET("/google-index/tasks/:id", middleware.AuthMiddleware(), middleware.AdminMiddleware(), googleIndexHandler.GetTaskStatus)
api.POST("/google-index/tasks/:id/start", middleware.AuthMiddleware(), middleware.AdminMiddleware(), googleIndexHandler.StartTask)
api.GET("/google-index/tasks/:id/items", middleware.AuthMiddleware(), middleware.AdminMiddleware(), googleIndexHandler.GetTaskItems)
// Google索引凭据上传和验证API
api.POST("/google-index/upload-credentials", middleware.AuthMiddleware(), middleware.AdminMiddleware(), googleIndexHandler.UploadCredentials)
api.POST("/google-index/validate-credentials", middleware.AuthMiddleware(), middleware.AdminMiddleware(), googleIndexHandler.ValidateCredentials)
}
// 设置监控系统
@@ -538,10 +565,11 @@ func main() {
// 静态文件服务
r.Static("/uploads", "./uploads")
r.Static("/data", "./data")
// 添加CORS头到静态文件
r.Use(func(c *gin.Context) {
if strings.HasPrefix(c.Request.URL.Path, "/uploads/") {
if strings.HasPrefix(c.Request.URL.Path, "/uploads/") || strings.HasPrefix(c.Request.URL.Path, "/data/") {
c.Header("Access-Control-Allow-Origin", "*")
c.Header("Access-Control-Allow-Methods", "GET, OPTIONS")
c.Header("Access-Control-Allow-Headers", "Origin, Content-Type, Accept")

241
pkg/google/client.go Normal file
View File

@@ -0,0 +1,241 @@
package google
import (
"context"
"encoding/json"
"fmt"
"os"
"time"
"golang.org/x/oauth2"
"golang.org/x/oauth2/google"
"google.golang.org/api/option"
"google.golang.org/api/searchconsole/v1"
)
// Client Google Search Console API客户端
type Client struct {
service *searchconsole.Service
SiteURL string
}
// Config 配置信息
type Config struct {
CredentialsFile string `json:"credentials_file"`
SiteURL string `json:"site_url"`
TokenFile string `json:"token_file"`
}
// URLInspectionRequest URL检查请求
type URLInspectionRequest struct {
InspectionURL string `json:"inspectionUrl"`
SiteURL string `json:"siteUrl"`
LanguageCode string `json:"languageCode"`
}
// URLInspectionResult URL检查结果
type URLInspectionResult struct {
IndexStatusResult struct {
IndexingState string `json:"indexingState"`
LastCrawled string `json:"lastCrawled"`
CrawlErrors []struct {
ErrorCode string `json:"errorCode"`
} `json:"crawlErrors"`
} `json:"indexStatusResult"`
MobileUsabilityResult struct {
MobileFriendly bool `json:"mobileFriendly"`
} `json:"mobileUsabilityResult"`
RichResultsResult struct {
Detected struct {
Items []struct {
RichResultType string `json:"richResultType"`
} `json:"items"`
} `json:"detected"`
} `json:"richResultsResult"`
}
// NewClient 创建新的客户端
func NewClient(config *Config) (*Client, error) {
ctx := context.Background()
// 读取认证文件
credentials, err := os.ReadFile(config.CredentialsFile)
if err != nil {
return nil, fmt.Errorf("读取认证文件失败: %v", err)
}
// 创建OAuth2配置
oauthConfig, err := google.ConfigFromJSON(credentials, searchconsole.WebmastersScope)
if err != nil {
return nil, fmt.Errorf("创建OAuth配置失败: %v", err)
}
// 尝试从文件读取token
token, err := tokenFromFile(config.TokenFile)
if err != nil {
// 如果没有token，启动web认证流程
token = getTokenFromWeb(oauthConfig)
saveToken(config.TokenFile, token)
}
// 创建HTTP客户端
client := oauthConfig.Client(ctx, token)
// 创建Search Console服务
service, err := searchconsole.NewService(ctx, option.WithHTTPClient(client))
if err != nil {
return nil, fmt.Errorf("创建Search Console服务失败: %v", err)
}
return &Client{
service: service,
SiteURL: config.SiteURL,
}, nil
}
// InspectURL 检查URL索引状态
func (c *Client) InspectURL(url string) (*URLInspectionResult, error) {
request := &searchconsole.InspectUrlIndexRequest{
InspectionUrl: url,
SiteUrl: c.SiteURL,
LanguageCode: "zh-CN",
}
call := c.service.UrlInspection.Index.Inspect(request)
response, err := call.Do()
if err != nil {
return nil, fmt.Errorf("检查URL失败: %v", err)
}
// 转换响应
result := &URLInspectionResult{}
if response.InspectionResult != nil {
if response.InspectionResult.IndexStatusResult != nil {
result.IndexStatusResult.IndexingState = string(response.InspectionResult.IndexStatusResult.IndexingState)
if response.InspectionResult.IndexStatusResult.LastCrawlTime != "" {
result.IndexStatusResult.LastCrawled = response.InspectionResult.IndexStatusResult.LastCrawlTime
}
}
if response.InspectionResult.MobileUsabilityResult != nil {
result.MobileUsabilityResult.MobileFriendly = response.InspectionResult.MobileUsabilityResult.Verdict == "MOBILE_USABILITY_VERdict_PASS"
}
if response.InspectionResult.RichResultsResult != nil && response.InspectionResult.RichResultsResult.Verdict != "RICH_RESULTS_VERdict_PASS" {
// 如果有富媒体结果检查信息
result.RichResultsResult.Detected.Items = append(result.RichResultsResult.Detected.Items, struct {
RichResultType string `json:"richResultType"`
}{
RichResultType: "UNKNOWN",
})
}
}
return result, nil
}
// SubmitSitemap 提交网站地图
func (c *Client) SubmitSitemap(sitemapURL string) error {
call := c.service.Sitemaps.Submit(c.SiteURL, sitemapURL)
err := call.Do()
if err != nil {
return fmt.Errorf("提交网站地图失败: %v", err)
}
return nil
}
// GetSites 获取已验证的网站列表
func (c *Client) GetSites() ([]*searchconsole.WmxSite, error) {
call := c.service.Sites.List()
response, err := call.Do()
if err != nil {
return nil, fmt.Errorf("获取网站列表失败: %v", err)
}
return response.SiteEntry, nil
}
// GetSearchAnalytics 获取搜索分析数据
func (c *Client) GetSearchAnalytics(startDate, endDate string) (*searchconsole.SearchAnalyticsQueryResponse, error) {
request := &searchconsole.SearchAnalyticsQueryRequest{
StartDate: startDate,
EndDate: endDate,
Type: "web",
}
call := c.service.Searchanalytics.Query(c.SiteURL, request)
response, err := call.Do()
if err != nil {
return nil, fmt.Errorf("获取搜索分析数据失败: %v", err)
}
return response, nil
}
// getTokenFromWeb 通过web流程获取token
func getTokenFromWeb(config *oauth2.Config) *oauth2.Token {
authURL := config.AuthCodeURL("state-token", oauth2.AccessTypeOffline)
fmt.Printf("请在浏览器中访问以下URL进行认证:\n%s\n", authURL)
fmt.Printf("输入授权代码: ")
var authCode string
if _, err := fmt.Scan(&authCode); err != nil {
panic(fmt.Sprintf("读取授权代码失败: %v", err))
}
token, err := config.Exchange(oauth2.NoContext, authCode)
if err != nil {
panic(fmt.Sprintf("获取token失败: %v", err))
}
return token
}
// tokenFromFile 从文件读取token
func tokenFromFile(file string) (*oauth2.Token, error) {
f, err := os.Open(file)
defer f.Close()
if err != nil {
return nil, err
}
token := &oauth2.Token{}
err = json.NewDecoder(f).Decode(token)
return token, err
}
// saveToken 保存token到文件
func saveToken(file string, token *oauth2.Token) {
fmt.Printf("保存凭证文件到: %s\n", file)
f, err := os.OpenFile(file, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
panic(fmt.Sprintf("无法保存凭证文件: %v", err))
}
defer f.Close()
json.NewEncoder(f).Encode(token)
}
// BatchInspectURL 批量检查URL状态
func (c *Client) BatchInspectURL(urls []string, callback func(url string, result *URLInspectionResult, err error)) {
semaphore := make(chan struct{}, 5) // 限制并发数
for _, url := range urls {
go func(u string) {
semaphore <- struct{}{} // 获取信号量
defer func() { <-semaphore }() // 释放信号量
result, err := c.InspectURL(u)
callback(u, result, err)
}(url)
// 避免请求过快
time.Sleep(100 * time.Millisecond)
}
// 等待所有goroutine完成
for i := 0; i < cap(semaphore); i++ {
semaphore <- struct{}{}
}
}

166
pkg/google/sitemap.go Normal file
View File

@@ -0,0 +1,166 @@
package google
import (
"encoding/xml"
"fmt"
"net/http"
"os"
"strings"
"time"
)
// Sitemap 网站地图结构
type Sitemap struct {
XMLName xml.Name `xml:"urlset"`
Xmlns string `xml:"xmlns,attr"`
URLs []SitemapURL `xml:"url"`
}
// SitemapURL 网站地图URL项
type SitemapURL struct {
Loc string `xml:"loc"`
LastMod string `xml:"lastmod,omitempty"`
ChangeFreq string `xml:"changefreq,omitempty"`
Priority string `xml:"priority,omitempty"`
}
// SitemapIndex 网站地图索引
type SitemapIndex struct {
XMLName xml.Name `xml:"sitemapindex"`
Xmlns string `xml:"xmlns,attr"`
Sitemaps []SitemapRef `xml:"sitemap"`
}
// SitemapRef 网站地图引用
type SitemapRef struct {
Loc string `xml:"loc"`
LastMod string `xml:"lastmod,omitempty"`
}
// GenerateSitemap 生成网站地图
func GenerateSitemap(urls []string, filename string) error {
sitemap := Sitemap{
Xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9",
}
for _, url := range urls {
sitemapURL := SitemapURL{
Loc: strings.TrimSpace(url),
LastMod: time.Now().Format("2006-01-02"),
ChangeFreq: "weekly",
Priority: "0.8",
}
sitemap.URLs = append(sitemap.URLs, sitemapURL)
}
data, err := xml.MarshalIndent(sitemap, "", " ")
if err != nil {
return fmt.Errorf("生成XML失败: %v", err)
}
// 添加XML头部
xmlData := []byte(xml.Header + string(data))
return os.WriteFile(filename, xmlData, 0644)
}
// GenerateSitemapIndex 生成网站地图索引
func GenerateSitemapIndex(sitemaps []string, filename string) error {
index := SitemapIndex{
Xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9",
}
for _, sitemap := range sitemaps {
ref := SitemapRef{
Loc: strings.TrimSpace(sitemap),
LastMod: time.Now().Format("2006-01-02"),
}
index.Sitemaps = append(index.Sitemaps, ref)
}
data, err := xml.MarshalIndent(index, "", " ")
if err != nil {
return fmt.Errorf("生成XML失败: %v", err)
}
// 添加XML头部
xmlData := []byte(xml.Header + string(data))
return os.WriteFile(filename, xmlData, 0644)
}
// SplitSitemap 将大量URL分割成多个网站地图
func SplitSitemap(urls []string, maxURLsPerSitemap int, baseURL string) ([]string, error) {
if maxURLsPerSitemap <= 0 {
maxURLsPerSitemap = 50000 // Google限制
}
var sitemapFiles []string
totalURLs := len(urls)
sitemapCount := (totalURLs + maxURLsPerSitemap - 1) / maxURLsPerSitemap
for i := 0; i < sitemapCount; i++ {
start := i * maxURLsPerSitemap
end := start + maxURLsPerSitemap
if end > totalURLs {
end = totalURLs
}
sitemapURLs := urls[start:end]
filename := fmt.Sprintf("sitemap_part_%d.xml", i+1)
err := GenerateSitemap(sitemapURLs, filename)
if err != nil {
return nil, fmt.Errorf("生成网站地图 %s 失败: %v", filename, err)
}
sitemapFiles = append(sitemapFiles, baseURL+filename)
fmt.Printf("生成网站地图: %s (%d URLs)\n", filename, len(sitemapURLs))
}
// 生成网站地图索引
if len(sitemapFiles) > 1 {
indexFiles := make([]string, len(sitemapFiles))
for i, file := range sitemapFiles {
indexFiles[i] = file
}
err := GenerateSitemapIndex(indexFiles, "sitemap_index.xml")
if err != nil {
return nil, fmt.Errorf("生成网站地图索引失败: %v", err)
}
fmt.Printf("生成网站地图索引: sitemap_index.xml\n")
}
return sitemapFiles, nil
}
// PingSearchEngines 通知搜索引擎网站地图更新
func PingSearchEngines(sitemapURL string) error {
searchEngines := []string{
"http://www.google.com/webmasters/sitemaps/ping?sitemap=",
"http://www.bing.com/webmaster/ping.aspx?siteMap=",
}
client := &http.Client{Timeout: 10 * time.Second}
for _, engine := range searchEngines {
fullURL := engine + sitemapURL
resp, err := client.Get(fullURL)
if err != nil {
fmt.Printf("通知搜索引擎失败 %s: %v\n", engine, err)
continue
}
resp.Body.Close()
if resp.StatusCode == 200 {
fmt.Printf("成功通知: %s\n", engine)
} else {
fmt.Printf("通知失败: %s (状态码: %d)\n", engine, resp.StatusCode)
}
}
return nil
}

396
task/google_index_processor.go Normal file
View File

@@ -0,0 +1,396 @@
package task
import (
"context"
"encoding/json"
"fmt"
"time"
"github.com/ctwj/urldb/db/entity"
"github.com/ctwj/urldb/db/repo"
"github.com/ctwj/urldb/pkg/google"
"github.com/ctwj/urldb/utils"
)
// GoogleIndexProcessor Google索引任务处理器
type GoogleIndexProcessor struct {
repoMgr *repo.RepositoryManager
client *google.Client
config *GoogleIndexProcessorConfig
}
// GoogleIndexProcessorConfig Google索引处理器配置
type GoogleIndexProcessorConfig struct {
CredentialsFile string
SiteURL string
TokenFile string
Concurrency int
RetryAttempts int
RetryDelay time.Duration
}
// GoogleIndexTaskInput Google索引任务输入数据结构
type GoogleIndexTaskInput struct {
URLs []string `json:"urls"`
Operation string `json:"operation"` // indexing_check, sitemap_submit, batch_index
SitemapURL string `json:"sitemap_url,omitempty"`
}
// GoogleIndexTaskOutput Google索引任务输出数据结构
type GoogleIndexTaskOutput struct {
URL string `json:"url,omitempty"`
IndexStatus string `json:"index_status,omitempty"`
Error string `json:"error,omitempty"`
Success bool `json:"success"`
Message string `json:"message"`
Time string `json:"time"`
Result *google.URLInspectionResult `json:"result,omitempty"`
}
// NewGoogleIndexProcessor 创建Google索引任务处理器
func NewGoogleIndexProcessor(repoMgr *repo.RepositoryManager) *GoogleIndexProcessor {
return &GoogleIndexProcessor{
repoMgr: repoMgr,
}
}
// GetTaskType 获取任务类型
func (gip *GoogleIndexProcessor) GetTaskType() string {
return "google_index"
}
// Process 处理Google索引任务项
func (gip *GoogleIndexProcessor) Process(ctx context.Context, taskID uint, item *entity.TaskItem) error {
utils.Info("开始处理Google索引任务项: %d", item.ID)
// 解析输入数据
var input GoogleIndexTaskInput
if err := json.Unmarshal([]byte(item.InputData), &input); err != nil {
utils.Error("解析输入数据失败: %v", err)
gip.updateTaskItemStatus(item, entity.TaskItemStatusFailed, "", false, nil, 400, err.Error())
return fmt.Errorf("解析输入数据失败: %v", err)
}
// 初始化Google客户端
client, err := gip.initGoogleClient()
if err != nil {
utils.Error("初始化Google客户端失败: %v", err)
gip.updateTaskItemStatus(item, entity.TaskItemStatusFailed, "", false, nil, 500, err.Error())
return fmt.Errorf("初始化Google客户端失败: %v", err)
}
// 根据操作类型执行不同任务
switch input.Operation {
case "url_indexing":
return gip.processURLIndexing(ctx, client, taskID, item, input)
case "sitemap_submit":
return gip.processSitemapSubmit(ctx, client, taskID, item, input)
case "status_check":
return gip.processStatusCheck(ctx, client, taskID, item, input)
default:
errorMsg := fmt.Sprintf("不支持的操作类型: %s", input.Operation)
gip.updateTaskItemStatus(item, entity.TaskItemStatusFailed, "", false, nil, 400, errorMsg)
return fmt.Errorf(errorMsg)
}
}
// processURLIndexing 处理URL索引检查
func (gip *GoogleIndexProcessor) processURLIndexing(ctx context.Context, client *google.Client, taskID uint, item *entity.TaskItem, input GoogleIndexTaskInput) error {
utils.Info("开始URL索引检查: %v", input.URLs)
for _, url := range input.URLs {
select {
case <-ctx.Done():
gip.updateTaskItemStatus(item, entity.TaskItemStatusFailed, "", false, nil, 0, "任务被取消")
return ctx.Err()
default:
// 检查URL索引状态
result, err := gip.inspectURL(client, url)
if err != nil {
utils.Error("检查URL索引状态失败: %s, 错误: %v", url, err)
gip.updateTaskItemStatus(item, entity.TaskItemStatusFailed, "", false, nil, 500, err.Error())
continue
}
// 更新任务项状态
var lastCrawled *time.Time
if result.IndexStatusResult.LastCrawled != "" {
parsedTime, err := time.Parse(time.RFC3339, result.IndexStatusResult.LastCrawled)
if err == nil {
lastCrawled = &parsedTime
}
}
gip.updateTaskItemStatus(item, entity.TaskItemStatusSuccess, result.IndexStatusResult.IndexingState, result.MobileUsabilityResult.MobileFriendly, lastCrawled, 200, "")
// 更新URL状态记录
gip.updateURLStatus(url, result.IndexStatusResult.IndexingState, lastCrawled)
// 添加延迟避免API限制
time.Sleep(100 * time.Millisecond)
}
}
utils.Info("URL索引检查完成")
return nil
}
// processSitemapSubmit 处理网站地图提交
func (gip *GoogleIndexProcessor) processSitemapSubmit(ctx context.Context, client *google.Client, taskID uint, item *entity.TaskItem, input GoogleIndexTaskInput) error {
utils.Info("开始网站地图提交: %s", input.SitemapURL)
if input.SitemapURL == "" {
errorMsg := "网站地图URL不能为空"
gip.updateTaskItemStatus(item, entity.TaskItemStatusFailed, "", false, nil, 400, errorMsg)
return fmt.Errorf(errorMsg)
}
// 提交网站地图
err := client.SubmitSitemap(input.SitemapURL)
if err != nil {
utils.Error("提交网站地图失败: %s, 错误: %v", input.SitemapURL, err)
gip.updateTaskItemStatus(item, entity.TaskItemStatusFailed, "", false, nil, 500, err.Error())
return fmt.Errorf("提交网站地图失败: %v", err)
}
// 更新任务项状态
now := time.Now()
gip.updateTaskItemStatus(item, entity.TaskItemStatusSuccess, "SUBMITTED", false, &now, 200, "")
utils.Info("网站地图提交完成: %s", input.SitemapURL)
return nil
}
// processStatusCheck 处理状态检查
func (gip *GoogleIndexProcessor) processStatusCheck(ctx context.Context, client *google.Client, taskID uint, item *entity.TaskItem, input GoogleIndexTaskInput) error {
utils.Info("开始状态检查: %v", input.URLs)
for _, url := range input.URLs {
select {
case <-ctx.Done():
gip.updateTaskItemStatus(item, entity.TaskItemStatusFailed, "", false, nil, 0, "任务被取消")
return ctx.Err()
default:
// 检查URL状态
result, err := gip.inspectURL(client, url)
if err != nil {
utils.Error("检查URL状态失败: %s, 错误: %v", url, err)
gip.updateTaskItemStatus(item, entity.TaskItemStatusFailed, "", false, nil, 500, err.Error())
continue
}
// 更新任务项状态
var lastCrawled *time.Time
if result.IndexStatusResult.LastCrawled != "" {
parsedTime, err := time.Parse(time.RFC3339, result.IndexStatusResult.LastCrawled)
if err == nil {
lastCrawled = &parsedTime
}
}
gip.updateTaskItemStatus(item, entity.TaskItemStatusSuccess, result.IndexStatusResult.IndexingState, result.MobileUsabilityResult.MobileFriendly, lastCrawled, 200, "")
utils.Info("URL状态检查完成: %s, 状态: %s", url, result.IndexStatusResult.IndexingState)
}
}
return nil
}
// initGoogleClient 初始化Google客户端
func (gip *GoogleIndexProcessor) initGoogleClient() (*google.Client, error) {
// 从配置中获取Google认证信息
credentialsFile, err := gip.repoMgr.SystemConfigRepository.GetConfigValue(entity.GoogleIndexConfigKeyCredentialsFile)
if err != nil || credentialsFile == "" {
return nil, fmt.Errorf("未配置Google认证文件: %v", err)
}
siteURL, err := gip.repoMgr.SystemConfigRepository.GetConfigValue(entity.GoogleIndexConfigKeySiteURL)
if err != nil || siteURL == "" {
return nil, fmt.Errorf("未配置网站URL: %v", err)
}
config := &google.Config{
CredentialsFile: credentialsFile,
SiteURL: siteURL,
TokenFile: "google_token.json", // 使用固定token文件名
}
client, err := google.NewClient(config)
if err != nil {
return nil, fmt.Errorf("创建Google客户端失败: %v", err)
}
return client, nil
}
// inspectURL 检查URL索引状态
func (gip *GoogleIndexProcessor) inspectURL(client *google.Client, url string) (*google.URLInspectionResult, error) {
// 重试机制
var result *google.URLInspectionResult
var err error
for attempt := 0; attempt <= gip.config.RetryAttempts; attempt++ {
result, err = client.InspectURL(url)
if err == nil {
break // 成功则退出重试循环
}
if attempt < gip.config.RetryAttempts {
utils.Info("URL检查失败，第%d次重试: %s, 错误: %v", attempt+1, url, err)
time.Sleep(gip.config.RetryDelay)
}
}
if err != nil {
return nil, fmt.Errorf("检查URL失败: %v", err)
}
return result, nil
}
// updateTaskItemStatus 更新任务项状态
func (gip *GoogleIndexProcessor) updateTaskItemStatus(item *entity.TaskItem, status entity.TaskItemStatus, indexStatus string, mobileFriendly bool, lastCrawled *time.Time, statusCode int, errorMessage string) {
item.Status = status
item.ErrorMessage = errorMessage
// 更新Google索引特有字段
item.IndexStatus = indexStatus
item.MobileFriendly = mobileFriendly
item.LastCrawled = lastCrawled
item.StatusCode = statusCode
now := time.Now()
item.ProcessedAt = &now
// 保存更新
if err := gip.repoMgr.TaskItemRepository.Update(item); err != nil {
utils.Error("更新任务项状态失败: %v", err)
}
}
// updateURLStatus 更新URL状态记录（使用任务项存储）
func (gip *GoogleIndexProcessor) updateURLStatus(url string, indexStatus string, lastCrawled *time.Time) {
// 在任务项中记录URL状态，而不是使用专门的URL状态表
// 此功能现在通过任务系统中的TaskItem记录来跟踪
utils.Debug("URL状态已更新: %s, 状态: %s", url, indexStatus)
}
// BatchProcessURLs 批量处理URLs
func (gip *GoogleIndexProcessor) BatchProcessURLs(ctx context.Context, urls []string, operation string, taskID uint) error {
utils.Info("开始批量处理URLs，数量: %d, 操作: %s", len(urls), operation)
// 根据并发数创建工作池
semaphore := make(chan struct{}, gip.config.Concurrency)
errChan := make(chan error, len(urls))
for _, url := range urls {
go func(u string) {
semaphore <- struct{}{} // 获取信号量
defer func() { <-semaphore }() // 释放信号量
// 处理单个URL
client, err := gip.initGoogleClient()
if err != nil {
errChan <- fmt.Errorf("初始化客户端失败: %v", err)
return
}
result, err := gip.inspectURL(client, u)
if err != nil {
utils.Error("处理URL失败: %s, 错误: %v", u, err)
errChan <- err
return
}
// 更新状态
var lastCrawled *time.Time
if result.IndexStatusResult.LastCrawled != "" {
parsedTime, err := time.Parse(time.RFC3339, result.IndexStatusResult.LastCrawled)
if err == nil {
lastCrawled = &parsedTime
}
}
// 创建任务项记录
now := time.Now()
inputData := map[string]interface{}{
"urls": []string{u},
"operation": "url_indexing",
}
inputDataJSON, _ := json.Marshal(inputData)
taskItem := &entity.TaskItem{
TaskID: taskID,
Status: entity.TaskItemStatusSuccess,
InputData: string(inputDataJSON),
URL: u,
IndexStatus: result.IndexStatusResult.IndexingState,
MobileFriendly: result.MobileUsabilityResult.MobileFriendly,
LastCrawled: lastCrawled,
StatusCode: 200,
ProcessedAt: &now,
}
if err := gip.repoMgr.TaskItemRepository.Create(taskItem); err != nil {
utils.Error("创建任务项失败: %v", err)
}
// 更新URL状态
gip.updateURLStatus(u, result.IndexStatusResult.IndexingState, lastCrawled)
errChan <- nil
}(url)
}
// 等待所有goroutine完成
for i := 0; i < len(urls); i++ {
err := <-errChan
if err != nil {
utils.Error("批量处理URL时出错: %v", err)
}
}
utils.Info("批量处理URLs完成")
return nil
}
// SubmitSitemap 提交网站地图
func (gip *GoogleIndexProcessor) SubmitSitemap(ctx context.Context, sitemapURL string, taskID uint) error {
utils.Info("开始提交网站地图: %s", sitemapURL)
client, err := gip.initGoogleClient()
if err != nil {
return fmt.Errorf("初始化Google客户端失败: %v", err)
}
err = client.SubmitSitemap(sitemapURL)
if err != nil {
return fmt.Errorf("提交网站地图失败: %v", err)
}
// 创建任务项记录
now := time.Now()
inputData := map[string]interface{}{
"sitemap_url": sitemapURL,
"operation": "sitemap_submit",
}
inputDataJSON, _ := json.Marshal(inputData)
taskItem := &entity.TaskItem{
TaskID: taskID,
Status: entity.TaskItemStatusSuccess,
InputData: string(inputDataJSON),
URL: sitemapURL,
IndexStatus: "SUBMITTED",
StatusCode: 200,
ProcessedAt: &now,
}
if err := gip.repoMgr.TaskItemRepository.Create(taskItem); err != nil {
utils.Error("创建任务项失败: %v", err)
}
utils.Info("网站地图提交完成: %s", sitemapURL)
return nil
}

68
task/task_processor.go
View File

@@ -555,3 +555,71 @@ func (tm *TaskManager) RecoverRunningTasks() error {
utils.Info("任务恢复完成，共恢复 %d 个任务", recoveredCount)
return nil
}
// CreateTask 创建任务
func (tm *TaskManager) CreateTask(taskType, name, description string, configID *uint) (*entity.Task, error) {
// 验证任务类型是否有对应的处理器
tm.mu.RLock()
_, exists := tm.processors[taskType]
tm.mu.RUnlock()
if !exists {
return nil, fmt.Errorf("未找到任务类型 %s 的处理器", taskType)
}
task := &entity.Task{
Type: entity.TaskType(taskType),
Name: name,
Title: name, // 设置Title为相同值，保持兼容性
Description: description,
Status: entity.TaskStatusPending,
ConfigID: configID,
}
err := tm.repoMgr.TaskRepository.Create(task)
if err != nil {
return nil, fmt.Errorf("创建任务失败: %v", err)
}
utils.Info("创建任务成功: ID=%d, 类型=%s, 名称=%s", task.ID, task.Type, task.Name)
return task, nil
}
// GetTask 获取任务详情
func (tm *TaskManager) GetTask(taskID uint) (*entity.Task, error) {
task, err := tm.repoMgr.TaskRepository.GetByID(taskID)
if err != nil {
return nil, fmt.Errorf("获取任务失败: %v", err)
}
return task, nil
}
// GetTaskItemStats 获取任务项统计信息
func (tm *TaskManager) GetTaskItemStats(taskID uint) (map[string]int, error) {
stats, err := tm.repoMgr.TaskItemRepository.GetStatsByTaskID(taskID)
if err != nil {
return nil, fmt.Errorf("获取任务项统计失败: %v", err)
}
return stats, nil
}
// QueryTaskItems 查询任务项
func (tm *TaskManager) QueryTaskItems(taskID uint, page, pageSize int, status string) ([]*entity.TaskItem, int64, error) {
items, total, err := tm.repoMgr.TaskItemRepository.GetListByTaskID(taskID, page, pageSize, status)
if err != nil {
return nil, 0, fmt.Errorf("查询任务项失败: %v", err)
}
return items, total, nil
}
// CreateTaskItems 创建任务项
func (tm *TaskManager) CreateTaskItems(taskID uint, items []*entity.TaskItem) error {
for _, item := range items {
item.TaskID = taskID
if err := tm.repoMgr.TaskItemRepository.Create(item); err != nil {
return fmt.Errorf("创建任务项失败: %v", err)
}
}
utils.Info("创建任务项成功: 任务ID=%d, 数量=%d", taskID, len(items))
return nil
}

4
web/composables/useApi.ts
View File

@@ -1,5 +1,6 @@
import { useApiFetch } from './useApiFetch'
import { useUserStore } from '~/stores/user'
import { useGoogleIndexApi } from './useGoogleIndexApi'
// 统一响应解析函数
export const parseApiResponse = <T>(response: any): T => {
@@ -447,6 +448,7 @@ export const useApi = () => {
apiAccessLogApi: useApiAccessLogApi(),
systemLogApi: useSystemLogApi(),
wechatApi: useWechatApi(),
sitemapApi: useSitemapApi()
sitemapApi: useSitemapApi(),
googleIndexApi: useGoogleIndexApi()
}
}

245
web/composables/useGoogleIndexApi.ts Normal file
View File

@@ -0,0 +1,245 @@
import { useApiFetch } from './useApiFetch'
import { parseApiResponse } from './useApi'
// Google索引配置类型定义
export interface GoogleIndexConfig {
id?: number
group: string
key: string
value: string
type?: string
}
// Google索引任务类型定义
export interface GoogleIndexTask {
id: number
title: string
type: string
status: string
description: string
totalItems: number
processedItems: number
successItems: number
failedItems: number
indexedURLs: number
failedURLs: number
errorMessage?: string
configID?: number
startedAt?: Date
completedAt?: Date
createdAt: Date
updatedAt: Date
}
// Google索引任务项类型定义
export interface GoogleIndexTaskItem {
id: number
taskID: number
URL: string
status: string
indexStatus: string
errorMessage?: string
inspectResult?: string
mobileFriendly: boolean
lastCrawled?: Date
statusCode: number
startedAt?: Date
completedAt?: Date
createdAt: Date
updatedAt: Date
}
// URL状态类型定义
export interface GoogleIndexURLStatus {
id: number
URL: string
indexStatus: string
lastChecked: Date
canonicalURL?: string
lastCrawled?: Date
changeFreq?: string
priority?: number
mobileFriendly: boolean
robotsBlocked: boolean
lastError?: string
statusCode: number
statusCodeText: string
checkCount: number
successCount: number
failureCount: number
createdAt: Date
updatedAt: Date
}
// Google索引状态响应类型定义
export interface GoogleIndexStatusResponse {
enabled: boolean
siteURL: string
lastCheckTime: Date
totalURLs: number
indexedURLs: number
notIndexedURLs: number
errorURLs: number
lastSitemapSubmit: Date
authValid: boolean
}
// Google索引任务列表响应类型定义
export interface GoogleIndexTaskListResponse {
tasks: GoogleIndexTask[]
total: number
page: number
pageSize: number
totalPages: number
}
// Google索引任务项分页响应类型定义
export interface GoogleIndexTaskItemPageResponse {
items: GoogleIndexTaskItem[]
total: number
page: number
size: number
}
// Google索引API封装
export const useGoogleIndexApi = () => {
// 配置管理API
const getGoogleIndexConfig = (params?: any) =>
useApiFetch('/google-index/config', { params }).then(parseApiResponse<GoogleIndexConfig[]>)
const getGoogleIndexConfigByKey = (key: string) =>
useApiFetch(`/google-index/config/${key}`).then(parseApiResponse<GoogleIndexConfig>)
const updateGoogleIndexConfig = (data: GoogleIndexConfig) =>
useApiFetch('/google-index/config', { method: 'POST', body: data }).then(parseApiResponse<GoogleIndexConfig>)
const deleteGoogleIndexConfig = (key: string) =>
useApiFetch(`/google-index/config/${key}`, { method: 'DELETE' }).then(parseApiResponse<boolean>)
// 任务管理API
const getGoogleIndexTasks = (params?: any) =>
useApiFetch('/google-index/tasks', { params }).then(parseApiResponse<GoogleIndexTaskListResponse>)
const getGoogleIndexTask = (id: number) =>
useApiFetch(`/google-index/tasks/${id}`).then(parseApiResponse<GoogleIndexTask>)
const createGoogleIndexTask = (data: any) =>
useApiFetch('/google-index/tasks', { method: 'POST', body: data }).then(parseApiResponse<GoogleIndexTask>)
const startGoogleIndexTask = (id: number) =>
useApiFetch(`/google-index/tasks/${id}/start`, { method: 'POST' }).then(parseApiResponse<boolean>)
const stopGoogleIndexTask = (id: number) =>
useApiFetch(`/google-index/tasks/${id}/stop`, { method: 'POST' }).then(parseApiResponse<boolean>)
const deleteGoogleIndexTask = (id: number) =>
useApiFetch(`/google-index/tasks/${id}`, { method: 'DELETE' }).then(parseApiResponse<boolean>)
// 任务项管理API
const getGoogleIndexTaskItems = (taskId: number, params?: any) =>
useApiFetch(`/google-index/tasks/${taskId}/items`, { params }).then(parseApiResponse<GoogleIndexTaskItemPageResponse>)
// URL状态管理API
const getGoogleIndexURLStatus = (params?: any) =>
useApiFetch('/google-index/urls/status', { params }).then(parseApiResponse<GoogleIndexURLStatus[]>)
const getGoogleIndexURLStatusByURL = (url: string) =>
useApiFetch(`/google-index/urls/status/${encodeURIComponent(url)}`).then(parseApiResponse<GoogleIndexURLStatus>)
const checkGoogleIndexURLStatus = (data: { urls: string[] }) =>
useApiFetch('/google-index/urls/check', { method: 'POST', body: data }).then(parseApiResponse<any>)
const submitGoogleIndexURL = (data: { urls: string[] }) =>
useApiFetch('/google-index/urls/submit', { method: 'POST', body: data }).then(parseApiResponse<any>)
// 批量操作API
const batchSubmitGoogleIndexURLs = (data: { urls: string[], operation: string }) =>
useApiFetch('/google-index/batch/submit', { method: 'POST', body: data }).then(parseApiResponse<any>)
const batchCheckGoogleIndexURLs = (data: { urls: string[], operation: string }) =>
useApiFetch('/google-index/batch/check', { method: 'POST', body: data }).then(parseApiResponse<any>)
// 网站地图提交API
const submitGoogleIndexSitemap = (data: { sitemapURL: string }) =>
useApiFetch('/google-index/sitemap/submit', { method: 'POST', body: data }).then(parseApiResponse<any>)
// 状态查询API
const getGoogleIndexStatus = () =>
useApiFetch('/google-index/status').then(parseApiResponse<GoogleIndexStatusResponse>)
// 验证凭据API
const validateCredentials = (data: { credentialsFile: string }) =>
useApiFetch('/google-index/validate-credentials', { method: 'POST', body: data }).then(parseApiResponse<any>)
// 更新Google索引分组配置API
const updateGoogleIndexGroupConfig = (data: GoogleIndexConfig) =>
useApiFetch('/google-index/config/update', { method: 'POST', body: data }).then(parseApiResponse<GoogleIndexConfig>)
// 上传凭据API
const uploadCredentials = (file: File) => {
const formData = new FormData()
formData.append('file', file)
return useApiFetch('/google-index/upload-credentials', {
method: 'POST',
body: formData,
headers: {
// 注意：此处不应包含Authorization头，因为文件上传通常由use-upload组件处理
}
}).then(parseApiResponse<any>)
}
// 调度器控制API
const startGoogleIndexScheduler = () =>
useApiFetch('/google-index/scheduler/start', { method: 'POST' }).then(parseApiResponse<boolean>)
const stopGoogleIndexScheduler = () =>
useApiFetch('/google-index/scheduler/stop', { method: 'POST' }).then(parseApiResponse<boolean>)
const getGoogleIndexSchedulerStatus = () =>
useApiFetch('/google-index/scheduler/status').then(parseApiResponse<any>)
return {
// 配置管理
getGoogleIndexConfig,
getGoogleIndexConfigByKey,
updateGoogleIndexConfig,
updateGoogleIndexGroupConfig,
deleteGoogleIndexConfig,
// 凭据验证和上传
validateCredentials,
uploadCredentials,
// 任务管理
getGoogleIndexTasks,
getGoogleIndexTask,
createGoogleIndexTask,
startGoogleIndexTask,
stopGoogleIndexTask,
deleteGoogleIndexTask,
// 任务项管理
getGoogleIndexTaskItems,
// URL状态管理
getGoogleIndexURLStatus,
getGoogleIndexURLStatusByURL,
checkGoogleIndexURLStatus,
submitGoogleIndexURL,
// 批量操作
batchSubmitGoogleIndexURLs,
batchCheckGoogleIndexURLs,
// 网站地图提交
submitGoogleIndexSitemap,
// 状态查询
getGoogleIndexStatus,
// 调度器控制
startGoogleIndexScheduler,
stopGoogleIndexScheduler,
getGoogleIndexSchedulerStatus
}
}

14
web/layouts/default.vue
View File

@@ -32,6 +32,20 @@
import { lightTheme } from 'naive-ui'
import { ref, onMounted } from 'vue'
// 动态添加Google站点验证meta标签
const { data: verificationData } = await $fetch('/api/public/site-verification').catch(() => ({ data: {} }))
useHead({
meta: verificationData?.google_site_verification_code
? [
{
name: 'google-site-verification',
content: verificationData.google_site_verification_code
}
]
: []
})
const theme = lightTheme
const isDark = ref(false)

47
web/pages/admin/index.vue
View File

@@ -22,7 +22,7 @@
</n-card>
<!-- 统计卡片 -->
<div class="grid grid-cols-1 md:grid-cols-3 gap-6">
<div class="grid grid-cols-1 md:grid-cols-4 gap-6">
<!-- 今日资源/总资源数 -->
<n-card>
<div class="flex items-center">
@@ -85,6 +85,27 @@
</n-button>
</template>
</n-card>
<!-- Google索引统计 -->
<n-card>
<div class="flex items-center">
<div class="p-3 bg-red-100 dark:bg-red-900 rounded-lg">
<i class="fab fa-google text-red-600 dark:text-red-400 text-xl"></i>
</div>
<div class="ml-4">
<p class="text-sm font-medium text-gray-600 dark:text-gray-400">已索引URL/总URL</p>
<p class="text-2xl font-bold text-gray-900 dark:text-white">{{ googleIndexStats.indexedURLs || 0 }}/{{ googleIndexStats.totalURLs || 0 }}</p>
</div>
</div>
<template #footer>
<n-button text type="primary" @click="navigateTo('/admin/seo#google-index')">
查看详情
<template #icon>
<i class="fas fa-arrow-right"></i>
</template>
</n-button>
</template>
</n-card>
</div>
<!-- 平台管理 -->
@@ -149,6 +170,17 @@ import { useApiFetch } from '~/composables/useApiFetch'
import { parseApiResponse } from '~/composables/useApi'
import Chart from 'chart.js/auto'
// Google索引统计
const googleIndexStats = ref({
totalURLs: 0,
indexedURLs: 0,
errorURLs: 0,
totalTasks: 0,
runningTasks: 0,
completedTasks: 0,
failedTasks: 0
})
// API
const statsApi = useStatsApi()
const panApi = usePanApi()
@@ -370,10 +402,23 @@ watch([weeklyViews, weeklySearches], () => {
})
})
// 加载Google索引统计
const loadGoogleIndexStats = async () => {
try {
const response = await useApiFetch('/google-index/status').then(parseApiResponse)
if (response && response.data) {
googleIndexStats.value = response.data
}
} catch (error) {
console.error('加载Google索引统计失败:', error)
}
}
// 组件挂载后初始化图表和数据
onMounted(async () => {
console.log('组件挂载，开始初始化...')
await fetchTrendData()
await loadGoogleIndexStats() // 加载Google索引统计
console.log('数据获取完成，准备初始化图表...')
nextTick(() => {
console.log('nextTick执行，开始初始化图表...')

1573
web/pages/admin/seo.vue
View File

File diff suppressed because it is too large Load Diff