mirror of
https://github.com/ctwj/urldb.git
synced 2025-11-25 03:15:04 +08:00
Compare commits
7 Commits
1af7fbd355
...
feat_googl
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4c6211d130 | ||
|
|
6f5b6c3f40 | ||
|
|
b893f30558 | ||
|
|
09baf0cb21 | ||
|
|
4e3f9017ac | ||
|
|
8a79e7e87b | ||
|
|
f67f820ef8 |
295
cmd/google-index/main.go
Normal file
295
cmd/google-index/main.go
Normal file
@@ -0,0 +1,295 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/ctwj/urldb/pkg/google"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 2 {
|
||||
printUsage()
|
||||
return
|
||||
}
|
||||
|
||||
command := os.Args[1]
|
||||
config := &google.Config{
|
||||
CredentialsFile: "credentials.json",
|
||||
SiteURL: "https://your-site.com/", // 替换为你的网站
|
||||
TokenFile: "token.json",
|
||||
}
|
||||
|
||||
client, err := google.NewClient(config)
|
||||
if err != nil {
|
||||
log.Fatalf("创建Google客户端失败: %v", err)
|
||||
}
|
||||
|
||||
switch command {
|
||||
case "inspect":
|
||||
if len(os.Args) < 3 {
|
||||
fmt.Println("请提供要检查的URL")
|
||||
return
|
||||
}
|
||||
inspectSingleURL(client, os.Args[2])
|
||||
|
||||
case "batch":
|
||||
if len(os.Args) < 3 {
|
||||
fmt.Println("请提供包含URL列表的文件")
|
||||
return
|
||||
}
|
||||
batchInspectURLs(client, os.Args[2])
|
||||
|
||||
case "sites":
|
||||
listSites(client)
|
||||
|
||||
case "analytics":
|
||||
getAnalytics(client)
|
||||
|
||||
case "sitemap":
|
||||
if len(os.Args) < 3 {
|
||||
fmt.Println("请提供网站地图URL")
|
||||
return
|
||||
}
|
||||
submitSitemap(client, os.Args[2])
|
||||
|
||||
default:
|
||||
fmt.Printf("未知命令: %s\n", command)
|
||||
printUsage()
|
||||
}
|
||||
}
|
||||
|
||||
func printUsage() {
|
||||
fmt.Println("Google Search Console API 工具")
|
||||
fmt.Println()
|
||||
fmt.Println("用法:")
|
||||
fmt.Println(" google-index inspect <url> - 检查单个URL的索引状态")
|
||||
fmt.Println(" google-index batch <file> - 批量检查URL状态")
|
||||
fmt.Println(" google-index sites - 列出已验证的网站")
|
||||
fmt.Println(" google-index analytics - 获取搜索分析数据")
|
||||
fmt.Println(" google-index sitemap <url> - 提交网站地图")
|
||||
fmt.Println()
|
||||
fmt.Println("配置:")
|
||||
fmt.Println(" - 创建 credentials.json 文件 (Google Cloud Console 下载)")
|
||||
fmt.Println(" - 修改 config.SiteURL 为你的网站URL")
|
||||
}
|
||||
|
||||
func inspectSingleURL(client *google.Client, url string) {
|
||||
fmt.Printf("正在检查URL: %s\n", url)
|
||||
|
||||
result, err := client.InspectURL(url)
|
||||
if err != nil {
|
||||
log.Printf("检查失败: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
printResult(url, result)
|
||||
}
|
||||
|
||||
func batchInspectURLs(client *google.Client, filename string) {
|
||||
urls, err := readURLsFromFile(filename)
|
||||
if err != nil {
|
||||
log.Fatalf("读取URL文件失败: %v", err)
|
||||
}
|
||||
|
||||
fmt.Printf("开始批量检查 %d 个URL...\n", len(urls))
|
||||
|
||||
results := make(chan struct {
|
||||
url string
|
||||
result *google.URLInspectionResult
|
||||
err error
|
||||
}, len(urls))
|
||||
|
||||
client.BatchInspectURL(urls, func(url string, result *google.URLInspectionResult, err error) {
|
||||
results <- struct {
|
||||
url string
|
||||
result *google.URLInspectionResult
|
||||
err error
|
||||
}{url, result, err}
|
||||
})
|
||||
|
||||
// 收集并打印结果
|
||||
fmt.Println("\n检查结果:")
|
||||
fmt.Println(strings.Repeat("-", 100))
|
||||
fmt.Printf("%-50s %-15s %-15s %-20s\n", "URL", "索引状态", "移动友好", "最后抓取")
|
||||
fmt.Println(strings.Repeat("-", 100))
|
||||
|
||||
for i := 0; i < len(urls); i++ {
|
||||
res := <-results
|
||||
if res.err != nil {
|
||||
fmt.Printf("%-50s %-15s\n", truncate(res.url, 47), "ERROR")
|
||||
continue
|
||||
}
|
||||
|
||||
indexStatus := res.result.IndexStatusResult.IndexingState
|
||||
mobileFriendly := "否"
|
||||
if res.result.MobileUsabilityResult.MobileFriendly {
|
||||
mobileFriendly = "是"
|
||||
}
|
||||
lastCrawl := res.result.IndexStatusResult.LastCrawled
|
||||
if lastCrawl == "" {
|
||||
lastCrawl = "未知"
|
||||
}
|
||||
|
||||
fmt.Printf("%-50s %-15s %-15s %-20s\n",
|
||||
truncate(res.url, 47), indexStatus, mobileFriendly, lastCrawl)
|
||||
}
|
||||
|
||||
fmt.Println(strings.Repeat("-", 100))
|
||||
}
|
||||
|
||||
func listSites(client *google.Client) {
|
||||
sites, err := client.GetSites()
|
||||
if err != nil {
|
||||
log.Printf("获取网站列表失败: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Println("已验证的网站:")
|
||||
fmt.Println(strings.Repeat("-", 80))
|
||||
fmt.Printf("%-50s %-15s %-15s\n", "网站URL", "权限级别", "验证状态")
|
||||
fmt.Println(strings.Repeat("-", 80))
|
||||
|
||||
for _, site := range sites {
|
||||
permissionLevel := string(site.PermissionLevel)
|
||||
verified := "否"
|
||||
if site.SiteUrl == client.SiteURL {
|
||||
verified = "是"
|
||||
}
|
||||
|
||||
fmt.Printf("%-50s %-15s %-15s\n",
|
||||
truncate(site.SiteUrl, 47), permissionLevel, verified)
|
||||
}
|
||||
|
||||
fmt.Println(strings.Repeat("-", 80))
|
||||
}
|
||||
|
||||
func getAnalytics(client *google.Client) {
|
||||
endDate := time.Now().Format("2006-01-02")
|
||||
startDate := time.Now().AddDate(0, -1, 0).Format("2006-01-02") // 最近30天
|
||||
|
||||
fmt.Printf("获取搜索分析数据 (%s 到 %s)...\n", startDate, endDate)
|
||||
|
||||
analytics, err := client.GetSearchAnalytics(startDate, endDate)
|
||||
if err != nil {
|
||||
log.Printf("获取分析数据失败: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
// 计算总计数据
|
||||
var totalClicks, totalImpressions float64
|
||||
var totalPosition float64
|
||||
|
||||
for _, row := range analytics.Rows {
|
||||
totalClicks += row.Clicks
|
||||
totalImpressions += row.Impressions
|
||||
totalPosition += row.Position
|
||||
}
|
||||
|
||||
avgCTR := float64(0)
|
||||
if totalImpressions > 0 {
|
||||
avgCTR = float64(totalClicks) / float64(totalImpressions) * 100
|
||||
}
|
||||
|
||||
avgPosition := float64(0)
|
||||
if len(analytics.Rows) > 0 {
|
||||
avgPosition = totalPosition / float64(len(analytics.Rows))
|
||||
}
|
||||
|
||||
fmt.Println("\n搜索分析摘要:")
|
||||
fmt.Println(strings.Repeat("-", 60))
|
||||
fmt.Printf("总点击数: %.0f\n", totalClicks)
|
||||
fmt.Printf("总展示次数: %.0f\n", totalImpressions)
|
||||
fmt.Printf("平均点击率: %.2f%%\n", avgCTR)
|
||||
fmt.Printf("平均排名: %.1f\n", avgPosition)
|
||||
fmt.Println(strings.Repeat("-", 60))
|
||||
|
||||
// 显示前10个页面
|
||||
if len(analytics.Rows) > 0 {
|
||||
fmt.Println("\n热门页面 (前10):")
|
||||
fmt.Printf("%-50s %-10s %-10s %-10s\n", "页面", "点击", "展示", "排名")
|
||||
fmt.Println(strings.Repeat("-", 80))
|
||||
|
||||
maxRows := len(analytics.Rows)
|
||||
if maxRows > 10 {
|
||||
maxRows = 10
|
||||
}
|
||||
|
||||
for i := 0; i < maxRows; i++ {
|
||||
row := analytics.Rows[i]
|
||||
fmt.Printf("%-50s %-10d %-10d %-10.1f\n",
|
||||
truncate(row.Keys[0], 47), row.Clicks, row.Impressions, row.Position)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func submitSitemap(client *google.Client, sitemapURL string) {
|
||||
fmt.Printf("正在提交网站地图: %s\n", sitemapURL)
|
||||
|
||||
err := client.SubmitSitemap(sitemapURL)
|
||||
if err != nil {
|
||||
log.Printf("提交网站地图失败: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Println("网站地图提交成功!")
|
||||
}
|
||||
|
||||
func readURLsFromFile(filename string) ([]string, error) {
|
||||
data, err := os.ReadFile(filename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
lines := strings.Split(string(data), "\n")
|
||||
var urls []string
|
||||
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if line != "" && !strings.HasPrefix(line, "#") {
|
||||
urls = append(urls, line)
|
||||
}
|
||||
}
|
||||
|
||||
return urls, nil
|
||||
}
|
||||
|
||||
func printResult(url string, result *google.URLInspectionResult) {
|
||||
fmt.Println("\nURL检查结果:")
|
||||
fmt.Println(strings.Repeat("=", 80))
|
||||
fmt.Printf("URL: %s\n", url)
|
||||
fmt.Println(strings.Repeat("-", 80))
|
||||
|
||||
fmt.Printf("索引状态: %s\n", result.IndexStatusResult.IndexingState)
|
||||
if result.IndexStatusResult.LastCrawled != "" {
|
||||
fmt.Printf("最后抓取: %s\n", result.IndexStatusResult.LastCrawled)
|
||||
}
|
||||
|
||||
fmt.Printf("移动友好: %t\n", result.MobileUsabilityResult.MobileFriendly)
|
||||
|
||||
if len(result.RichResultsResult.Detected.Items) > 0 {
|
||||
fmt.Println("富媒体结果:")
|
||||
for _, item := range result.RichResultsResult.Detected.Items {
|
||||
fmt.Printf(" - %s\n", item.RichResultType)
|
||||
}
|
||||
}
|
||||
|
||||
if len(result.IndexStatusResult.CrawlErrors) > 0 {
|
||||
fmt.Println("抓取错误:")
|
||||
for _, err := range result.IndexStatusResult.CrawlErrors {
|
||||
fmt.Printf(" - %s\n", err.ErrorCode)
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Println(strings.Repeat("=", 80))
|
||||
}
|
||||
|
||||
func truncate(s string, length int) string {
|
||||
if len(s) <= length {
|
||||
return s
|
||||
}
|
||||
return s[:length-3] + "..."
|
||||
}
|
||||
@@ -219,7 +219,19 @@ func createIndexes(db *gorm.DB) {
|
||||
db.Exec("CREATE INDEX IF NOT EXISTS idx_api_access_logs_response_time ON api_access_logs(processing_time)")
|
||||
db.Exec("CREATE INDEX IF NOT EXISTS idx_api_access_logs_error_logs ON api_access_logs(response_status, created_at DESC) WHERE response_status >= 400")
|
||||
|
||||
utils.Info("数据库索引创建完成(已移除全文搜索索引,准备使用Meilisearch,新增API访问日志性能索引)")
|
||||
// 任务和任务项表索引 - Google索引功能优化
|
||||
db.Exec("CREATE INDEX IF NOT EXISTS idx_tasks_type_status ON tasks(type, status)")
|
||||
db.Exec("CREATE INDEX IF NOT EXISTS idx_tasks_created_at ON tasks(created_at DESC)")
|
||||
|
||||
// task_items表的关键索引 - 支持高效去重和状态查询
|
||||
db.Exec("CREATE INDEX IF NOT EXISTS idx_task_items_url ON task_items(url)")
|
||||
db.Exec("CREATE INDEX IF NOT EXISTS idx_task_items_url_status ON task_items(url, status)")
|
||||
db.Exec("CREATE INDEX IF NOT EXISTS idx_task_items_task_id ON task_items(task_id)")
|
||||
db.Exec("CREATE INDEX IF NOT EXISTS idx_task_items_status ON task_items(status)")
|
||||
db.Exec("CREATE INDEX IF NOT EXISTS idx_task_items_created_at ON task_items(created_at DESC)")
|
||||
db.Exec("CREATE INDEX IF NOT EXISTS idx_task_items_status_created ON task_items(status, created_at)")
|
||||
|
||||
utils.Info("数据库索引创建完成(已移除全文搜索索引,准备使用Meilisearch,新增API访问日志性能索引,任务项表索引优化)")
|
||||
}
|
||||
|
||||
// insertDefaultDataIfEmpty 只在数据库为空时插入默认数据
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package converter
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"time"
|
||||
|
||||
@@ -325,3 +326,38 @@ func ToReadyResourceResponseList(resources []entity.ReadyResource) []dto.ReadyRe
|
||||
// Key: req.Key,
|
||||
// }
|
||||
// }
|
||||
|
||||
// TaskToGoogleIndexTaskOutput 将Task实体转换为GoogleIndexTaskOutput
|
||||
func TaskToGoogleIndexTaskOutput(task *entity.Task, stats map[string]int) dto.GoogleIndexTaskOutput {
|
||||
output := dto.GoogleIndexTaskOutput{
|
||||
ID: task.ID,
|
||||
Name: task.Name,
|
||||
Description: task.Description,
|
||||
Type: string(task.Type),
|
||||
Status: string(task.Status),
|
||||
Progress: task.Progress,
|
||||
TotalItems: stats["total"],
|
||||
ProcessedItems: stats["completed"] + stats["failed"],
|
||||
SuccessfulItems: stats["completed"],
|
||||
FailedItems: stats["failed"],
|
||||
PendingItems: stats["pending"],
|
||||
ProcessingItems: stats["processing"],
|
||||
IndexedURLs: task.IndexedURLs,
|
||||
FailedURLs: task.FailedURLs,
|
||||
ConfigID: task.ConfigID,
|
||||
CreatedAt: task.CreatedAt,
|
||||
UpdatedAt: task.UpdatedAt,
|
||||
StartedAt: task.StartedAt,
|
||||
CompletedAt: task.CompletedAt,
|
||||
}
|
||||
|
||||
// 设置进度数据
|
||||
if task.ProgressData != "" {
|
||||
var progressData map[string]interface{}
|
||||
if err := json.Unmarshal([]byte(task.ProgressData), &progressData); err == nil {
|
||||
output.ProgressData = progressData
|
||||
}
|
||||
}
|
||||
|
||||
return output
|
||||
}
|
||||
|
||||
@@ -305,7 +305,7 @@ func SystemConfigToPublicResponse(configs []entity.SystemConfig) map[string]inte
|
||||
entity.ConfigResponseFieldEnableRegister: true, // 默认开启注册功能
|
||||
entity.ConfigResponseFieldThirdPartyStatsCode: "",
|
||||
entity.ConfigResponseFieldWebsiteURL: "",
|
||||
}
|
||||
}
|
||||
|
||||
// 将键值对转换为map,过滤掉敏感配置
|
||||
for _, config := range configs {
|
||||
|
||||
188
db/dto/google_index.go
Normal file
188
db/dto/google_index.go
Normal file
@@ -0,0 +1,188 @@
|
||||
package dto
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// GoogleIndexConfigInput Google索引配置输入
|
||||
type GoogleIndexConfigInput struct {
|
||||
Group string `json:"group" binding:"required"`
|
||||
Key string `json:"key" binding:"required"`
|
||||
Value string `json:"value" binding:"required"`
|
||||
Type string `json:"type" default:"string"`
|
||||
}
|
||||
|
||||
// GoogleIndexConfigOutput Google索引配置输出
|
||||
type GoogleIndexConfigOutput struct {
|
||||
ID uint `json:"id"`
|
||||
Group string `json:"group"`
|
||||
Key string `json:"key"`
|
||||
Value string `json:"value"`
|
||||
Type string `json:"type"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// GoogleIndexConfigGeneral 通用配置
|
||||
type GoogleIndexConfigGeneral struct {
|
||||
Enabled bool `json:"enabled" binding:"required"`
|
||||
SiteURL string `json:"site_url" binding:"required"`
|
||||
SiteName string `json:"site_name"`
|
||||
}
|
||||
|
||||
// GoogleIndexConfigAuth 认证配置
|
||||
type GoogleIndexConfigAuth struct {
|
||||
CredentialsFile string `json:"credentials_file"`
|
||||
ClientEmail string `json:"client_email"`
|
||||
ClientID string `json:"client_id"`
|
||||
PrivateKey string `json:"private_key"`
|
||||
Token string `json:"token"`
|
||||
}
|
||||
|
||||
// GoogleIndexConfigSchedule 调度配置
|
||||
type GoogleIndexConfigSchedule struct {
|
||||
CheckInterval int `json:"check_interval" binding:"required,min=1,max=1440"` // 检查间隔(分钟),1-24小时
|
||||
BatchSize int `json:"batch_size" binding:"required,min=1,max=1000"` // 批处理大小
|
||||
Concurrency int `json:"concurrency" binding:"required,min=1,max=10"` // 并发数
|
||||
RetryAttempts int `json:"retry_attempts" binding:"required,min=0,max=10"` // 重试次数
|
||||
RetryDelay int `json:"retry_delay" binding:"required,min=1,max=60"` // 重试延迟(秒)
|
||||
}
|
||||
|
||||
// GoogleIndexConfigSitemap 网站地图配置
|
||||
type GoogleIndexConfigSitemap struct {
|
||||
AutoSitemap bool `json:"auto_sitemap"`
|
||||
SitemapPath string `json:"sitemap_path" default:"/sitemap.xml"`
|
||||
SitemapSchedule string `json:"sitemap_schedule" default:"@daily"` // cron表达式
|
||||
}
|
||||
|
||||
// GoogleIndexTaskInput Google索引任务输入
|
||||
type GoogleIndexTaskInput struct {
|
||||
Title string `json:"title" binding:"required"`
|
||||
Type string `json:"type" binding:"required"`
|
||||
Description string `json:"description"`
|
||||
URLs []string `json:"urls,omitempty"` // 用于URL索引检查任务
|
||||
SitemapURL string `json:"sitemap_url,omitempty"` // 用于网站地图提交任务
|
||||
ConfigID *uint `json:"config_id,omitempty"`
|
||||
}
|
||||
|
||||
// GoogleIndexTaskOutput Google索引任务输出
|
||||
type GoogleIndexTaskOutput struct {
|
||||
ID uint `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
Type string `json:"type"`
|
||||
Status string `json:"status"`
|
||||
Progress float64 `json:"progress"`
|
||||
TotalItems int `json:"total_items"`
|
||||
ProcessedItems int `json:"processed_items"`
|
||||
SuccessfulItems int `json:"successful_items"`
|
||||
FailedItems int `json:"failed_items"`
|
||||
PendingItems int `json:"pending_items"`
|
||||
ProcessingItems int `json:"processing_items"`
|
||||
IndexedURLs int `json:"indexed_urls"`
|
||||
FailedURLs int `json:"failed_urls"`
|
||||
ConfigID *uint `json:"config_id"`
|
||||
ProgressData map[string]interface{} `json:"progress_data"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
StartedAt *time.Time `json:"started_at"`
|
||||
CompletedAt *time.Time `json:"completed_at"`
|
||||
}
|
||||
|
||||
// GoogleIndexTaskItemInput Google索引任务项输入
|
||||
type GoogleIndexTaskItemInput struct {
|
||||
TaskID uint `json:"task_id" binding:"required"`
|
||||
URL string `json:"url" binding:"required,url"`
|
||||
}
|
||||
|
||||
// GoogleIndexTaskItemOutput Google索引任务项输出
|
||||
type GoogleIndexTaskItemOutput struct {
|
||||
ID uint `json:"id"`
|
||||
TaskID uint `json:"task_id"`
|
||||
URL string `json:"url"`
|
||||
Status string `json:"status"`
|
||||
IndexStatus string `json:"index_status"`
|
||||
ErrorMessage string `json:"error_message"`
|
||||
InspectResult string `json:"inspect_result"`
|
||||
MobileFriendly bool `json:"mobile_friendly"`
|
||||
LastCrawled *time.Time `json:"last_crawled"`
|
||||
StatusCode int `json:"status_code"`
|
||||
StartedAt *time.Time `json:"started_at"`
|
||||
CompletedAt *time.Time `json:"completed_at"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// GoogleIndexURLStatusInput Google索引URL状态输入
|
||||
type GoogleIndexURLStatusInput struct {
|
||||
URL string `json:"url" binding:"required,url"`
|
||||
IndexStatus string `json:"index_status" binding:"required"`
|
||||
}
|
||||
|
||||
// GoogleIndexURLStatusOutput Google索引URL状态输出
|
||||
type GoogleIndexURLStatusOutput struct {
|
||||
ID uint `json:"id"`
|
||||
URL string `json:"url"`
|
||||
IndexStatus string `json:"index_status"`
|
||||
LastChecked time.Time `json:"last_checked"`
|
||||
CanonicalURL *string `json:"canonical_url"`
|
||||
LastCrawled *time.Time `json:"last_crawled"`
|
||||
ChangeFreq *string `json:"change_freq"`
|
||||
Priority *float64 `json:"priority"`
|
||||
MobileFriendly bool `json:"mobile_friendly"`
|
||||
RobotsBlocked bool `json:"robots_blocked"`
|
||||
LastError *string `json:"last_error"`
|
||||
StatusCode int `json:"status_code"`
|
||||
StatusCodeText string `json:"status_code_text"`
|
||||
CheckCount int `json:"check_count"`
|
||||
SuccessCount int `json:"success_count"`
|
||||
FailureCount int `json:"failure_count"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// GoogleIndexBatchRequest 批量处理请求
|
||||
type GoogleIndexBatchRequest struct {
|
||||
URLs []string `json:"urls" binding:"required,min=1,max=1000"`
|
||||
Operation string `json:"operation" binding:"required,oneof=check_index submit_sitemap ping"` // 操作类型
|
||||
}
|
||||
|
||||
// GoogleIndexBatchResponse 批量处理响应
|
||||
type GoogleIndexBatchResponse struct {
|
||||
Success bool `json:"success"`
|
||||
Results []string `json:"results,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
Total int `json:"total"`
|
||||
Processed int `json:"processed"`
|
||||
Failed int `json:"failed"`
|
||||
}
|
||||
|
||||
// GoogleIndexStatusResponse 索引状态响应
|
||||
type GoogleIndexStatusResponse struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
SiteURL string `json:"site_url"`
|
||||
LastCheckTime time.Time `json:"last_check_time"`
|
||||
TotalURLs int `json:"total_urls"`
|
||||
IndexedURLs int `json:"indexed_urls"`
|
||||
NotIndexedURLs int `json:"not_indexed_urls"`
|
||||
ErrorURLs int `json:"error_urls"`
|
||||
LastSitemapSubmit time.Time `json:"last_sitemap_submit"`
|
||||
AuthValid bool `json:"auth_valid"`
|
||||
}
|
||||
|
||||
// GoogleIndexTaskListResponse 任务列表响应
|
||||
type GoogleIndexTaskListResponse struct {
|
||||
Tasks []GoogleIndexTaskOutput `json:"tasks"`
|
||||
Total int64 `json:"total"`
|
||||
Page int `json:"page"`
|
||||
PageSize int `json:"page_size"`
|
||||
TotalPages int `json:"total_pages"`
|
||||
}
|
||||
|
||||
// GoogleIndexTaskItemPageResponse 任务项分页响应
|
||||
type GoogleIndexTaskItemPageResponse struct {
|
||||
Items []GoogleIndexTaskItemOutput `json:"items"`
|
||||
Total int64 `json:"total"`
|
||||
Page int `json:"page"`
|
||||
Size int `json:"size"`
|
||||
}
|
||||
@@ -132,3 +132,32 @@ type SearchRequest struct {
|
||||
Page int `json:"page"`
|
||||
Limit int `json:"limit"`
|
||||
}
|
||||
|
||||
// CreateTaskRequest 创建任务请求
|
||||
type CreateTaskRequest struct {
|
||||
Name string `json:"name" binding:"required"`
|
||||
Description string `json:"description"`
|
||||
TaskType string `json:"task_type" binding:"required"`
|
||||
ConfigID *uint `json:"config_id"`
|
||||
}
|
||||
|
||||
// CreateTaskItemRequest 创建任务项请求
|
||||
type CreateTaskItemRequest struct {
|
||||
URL string `json:"url"`
|
||||
InputData map[string]interface{} `json:"input_data"`
|
||||
}
|
||||
|
||||
// QueryTaskRequest 查询任务请求
|
||||
type QueryTaskRequest struct {
|
||||
Page int `json:"page" form:"page"`
|
||||
PageSize int `json:"page_size" form:"page_size"`
|
||||
Status string `json:"status" form:"status"`
|
||||
Type string `json:"type" form:"type"`
|
||||
}
|
||||
|
||||
// QueryTaskItemRequest 查询任务项请求
|
||||
type QueryTaskItemRequest struct {
|
||||
Page int `json:"page" form:"page"`
|
||||
PageSize int `json:"page_size" form:"page_size"`
|
||||
Status string `json:"status" form:"status"`
|
||||
}
|
||||
|
||||
28
db/entity/google_index_config_constants.go
Normal file
28
db/entity/google_index_config_constants.go
Normal file
@@ -0,0 +1,28 @@
|
||||
package entity
|
||||
|
||||
// GoogleIndexConfigKeys Google索引配置键常量
|
||||
const (
|
||||
// 通用配置
|
||||
GoogleIndexConfigKeyEnabled = "google_index_enabled" // 是否启用Google索引功能
|
||||
GoogleIndexConfigKeySiteURL = "google_index_site_url" // 网站URL
|
||||
GoogleIndexConfigKeySiteName = "google_index_site_name" // 网站名称
|
||||
|
||||
// 认证配置
|
||||
GoogleIndexConfigKeyCredentialsFile = "google_index_credentials_file" // 凭证文件路径
|
||||
GoogleIndexConfigKeyClientEmail = "google_index_client_email" // 客户端邮箱
|
||||
GoogleIndexConfigKeyClientID = "google_index_client_id" // 客户端ID
|
||||
GoogleIndexConfigKeyPrivateKey = "google_index_private_key" // 私钥(加密存储)
|
||||
GoogleIndexConfigKeyToken = "google_index_token" // 访问令牌
|
||||
|
||||
// 调度配置
|
||||
GoogleIndexConfigKeyCheckInterval = "google_index_check_interval" // 检查间隔(分钟)
|
||||
GoogleIndexConfigKeyBatchSize = "google_index_batch_size" // 批处理大小
|
||||
GoogleIndexConfigKeyConcurrency = "google_index_concurrency" // 并发数
|
||||
GoogleIndexConfigKeyRetryAttempts = "google_index_retry_attempts" // 重试次数
|
||||
GoogleIndexConfigKeyRetryDelay = "google_index_retry_delay" // 重试延迟(秒)
|
||||
|
||||
// 网站地图配置
|
||||
GoogleIndexConfigKeyAutoSitemap = "google_index_auto_sitemap" // 自动提交网站地图
|
||||
GoogleIndexConfigKeySitemapPath = "google_index_sitemap_path" // 网站地图路径
|
||||
GoogleIndexConfigKeySitemapSchedule = "google_index_sitemap_schedule" // 网站地图调度
|
||||
)
|
||||
@@ -250,5 +250,5 @@ const (
|
||||
ConfigDefaultQrCodeStyle = "Plain"
|
||||
|
||||
// 网站URL配置默认值
|
||||
ConfigDefaultWebsiteURL = ""
|
||||
ConfigDefaultWebsiteURL = "https://example.com"
|
||||
)
|
||||
|
||||
@@ -24,21 +24,24 @@ type TaskType string
|
||||
const (
|
||||
TaskTypeBatchTransfer TaskType = "batch_transfer" // 批量转存
|
||||
TaskTypeExpansion TaskType = "expansion" // 账号扩容
|
||||
TaskTypeGoogleIndex TaskType = "google_index" // Google索引
|
||||
)
|
||||
|
||||
// Task 任务表
|
||||
type Task struct {
|
||||
ID uint `json:"id" gorm:"primaryKey;autoIncrement"`
|
||||
Title string `json:"title" gorm:"size:255;not null;comment:任务标题"`
|
||||
Name string `json:"name" gorm:"size:255;not null;default:'';comment:任务名称"`
|
||||
Type TaskType `json:"type" gorm:"size:50;not null;comment:任务类型"`
|
||||
Status TaskStatus `json:"status" gorm:"size:20;not null;default:pending;comment:任务状态"`
|
||||
Description string `json:"description" gorm:"type:text;comment:任务描述"`
|
||||
|
||||
// 进度信息
|
||||
TotalItems int `json:"total_items" gorm:"not null;default:0;comment:总项目数"`
|
||||
ProcessedItems int `json:"processed_items" gorm:"not null;default:0;comment:已处理项目数"`
|
||||
SuccessItems int `json:"success_items" gorm:"not null;default:0;comment:成功项目数"`
|
||||
FailedItems int `json:"failed_items" gorm:"not null;default:0;comment:失败项目数"`
|
||||
TotalItems int `json:"total_items" gorm:"not null;default:0;comment:总项目数"`
|
||||
ProcessedItems int `json:"processed_items" gorm:"not null;default:0;comment:已处理项目数"`
|
||||
SuccessItems int `json:"success_items" gorm:"not null;default:0;comment:成功项目数"`
|
||||
FailedItems int `json:"failed_items" gorm:"not null;default:0;comment:失败项目数"`
|
||||
Progress float64 `json:"progress" gorm:"not null;default:0.0;comment:任务进度"`
|
||||
|
||||
// 任务配置 (JSON格式存储)
|
||||
Config string `json:"config" gorm:"type:text;comment:任务配置"`
|
||||
@@ -46,6 +49,16 @@ type Task struct {
|
||||
// 任务消息
|
||||
Message string `json:"message" gorm:"type:text;comment:任务消息"`
|
||||
|
||||
// 进度数据 (JSON格式存储)
|
||||
ProgressData string `json:"progress_data" gorm:"type:text;comment:进度数据"`
|
||||
|
||||
// Google索引特有字段 (当Type为google_index时使用)
|
||||
IndexedURLs int `json:"indexed_urls" gorm:"default:0;comment:已索引URL数量"`
|
||||
FailedURLs int `json:"failed_urls" gorm:"default:0;comment:失败URL数量"`
|
||||
|
||||
// 配置关联 (用于Google索引任务)
|
||||
ConfigID *uint `json:"config_id" gorm:"comment:配置ID"`
|
||||
|
||||
// 时间信息
|
||||
StartedAt *time.Time `json:"started_at" gorm:"comment:开始时间"`
|
||||
CompletedAt *time.Time `json:"completed_at" gorm:"comment:完成时间"`
|
||||
|
||||
@@ -35,6 +35,14 @@ type TaskItem struct {
|
||||
// 处理日志 (可选,用于记录详细的处理过程)
|
||||
ProcessLog string `json:"process_log" gorm:"type:text;comment:处理日志"`
|
||||
|
||||
// Google索引特有字段 (当任务类型为google_index时使用)
|
||||
URL string `json:"url" gorm:"size:2048;comment:URL (Google索引专用)"`
|
||||
IndexStatus string `json:"index_status" gorm:"size:50;comment:索引状态 (Google索引专用)"`
|
||||
InspectResult string `json:"inspect_result" gorm:"type:text;comment:检查结果 (Google索引专用)"`
|
||||
MobileFriendly bool `json:"mobile_friendly" gorm:"default:false;comment:是否移动友好 (Google索引专用)"`
|
||||
LastCrawled *time.Time `json:"last_crawled" gorm:"comment:最后抓取时间 (Google索引专用)"`
|
||||
StatusCode int `json:"status_code" gorm:"default:0;comment:HTTP状态码 (Google索引专用)"`
|
||||
|
||||
// 时间信息
|
||||
ProcessedAt *time.Time `json:"processed_at" gorm:"comment:处理时间"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"autoCreateTime"`
|
||||
|
||||
@@ -51,6 +51,8 @@ type ResourceRepository interface {
|
||||
FindByResourceKey(key string) ([]entity.Resource, error)
|
||||
FindByKey(key string) ([]entity.Resource, error)
|
||||
GetHotResources(limit int) ([]entity.Resource, error)
|
||||
GetTotalCount() (int64, error)
|
||||
GetAllValidResources() ([]entity.Resource, error)
|
||||
}
|
||||
|
||||
// ResourceRepositoryImpl Resource的Repository实现
|
||||
@@ -799,3 +801,18 @@ func (r *ResourceRepositoryImpl) FindByResourceKey(key string) ([]entity.Resourc
|
||||
}
|
||||
return resources, nil
|
||||
}
|
||||
|
||||
// GetTotalCount 获取资源总数
|
||||
func (r *ResourceRepositoryImpl) GetTotalCount() (int64, error) {
|
||||
var count int64
|
||||
err := r.GetDB().Model(&entity.Resource{}).Count(&count).Error
|
||||
return count, err
|
||||
}
|
||||
|
||||
// GetAllValidResources 获取所有有效的公开资源
|
||||
func (r *ResourceRepositoryImpl) GetAllValidResources() ([]entity.Resource, error) {
|
||||
var resources []entity.Resource
|
||||
err := r.GetDB().Where("is_valid = ? AND is_public = ?", true, true).
|
||||
Find(&resources).Error
|
||||
return resources, err
|
||||
}
|
||||
|
||||
@@ -144,6 +144,16 @@ func (r *SystemConfigRepositoryImpl) GetOrCreateDefault() ([]entity.SystemConfig
|
||||
{Key: entity.ConfigKeyTelegramQrImage, Value: entity.ConfigDefaultTelegramQrImage, Type: entity.ConfigTypeString},
|
||||
{Key: entity.ConfigKeyQrCodeStyle, Value: entity.ConfigDefaultQrCodeStyle, Type: entity.ConfigTypeString},
|
||||
{Key: entity.ConfigKeyWebsiteURL, Value: entity.ConfigDefaultWebsiteURL, Type: entity.ConfigTypeString},
|
||||
// Google索引配置
|
||||
{Key: entity.GoogleIndexConfigKeyEnabled, Value: "false", Type: entity.ConfigTypeBool},
|
||||
{Key: entity.GoogleIndexConfigKeySiteName, Value: entity.ConfigDefaultSiteTitle, Type: entity.ConfigTypeString},
|
||||
{Key: entity.GoogleIndexConfigKeyCheckInterval, Value: "60", Type: entity.ConfigTypeInt},
|
||||
{Key: entity.GoogleIndexConfigKeyBatchSize, Value: "10", Type: entity.ConfigTypeInt},
|
||||
{Key: entity.GoogleIndexConfigKeyConcurrency, Value: "2", Type: entity.ConfigTypeInt},
|
||||
{Key: entity.GoogleIndexConfigKeyRetryAttempts, Value: "3", Type: entity.ConfigTypeInt},
|
||||
{Key: entity.GoogleIndexConfigKeyRetryDelay, Value: "2", Type: entity.ConfigTypeInt},
|
||||
{Key: entity.GoogleIndexConfigKeyAutoSitemap, Value: "false", Type: entity.ConfigTypeBool},
|
||||
{Key: entity.GoogleIndexConfigKeySitemapPath, Value: "/sitemap.xml", Type: entity.ConfigTypeString},
|
||||
}
|
||||
|
||||
createStart := utils.GetCurrentTime()
|
||||
@@ -191,6 +201,16 @@ func (r *SystemConfigRepositoryImpl) GetOrCreateDefault() ([]entity.SystemConfig
|
||||
entity.ConfigKeyWechatSearchImage: {Key: entity.ConfigKeyWechatSearchImage, Value: entity.ConfigDefaultWechatSearchImage, Type: entity.ConfigTypeString},
|
||||
entity.ConfigKeyTelegramQrImage: {Key: entity.ConfigKeyTelegramQrImage, Value: entity.ConfigDefaultTelegramQrImage, Type: entity.ConfigTypeString},
|
||||
entity.ConfigKeyWebsiteURL: {Key: entity.ConfigKeyWebsiteURL, Value: entity.ConfigDefaultWebsiteURL, Type: entity.ConfigTypeString},
|
||||
// Google索引配置
|
||||
entity.GoogleIndexConfigKeyEnabled: {Key: entity.GoogleIndexConfigKeyEnabled, Value: "false", Type: entity.ConfigTypeBool},
|
||||
entity.GoogleIndexConfigKeySiteName: {Key: entity.GoogleIndexConfigKeySiteName, Value: entity.ConfigDefaultSiteTitle, Type: entity.ConfigTypeString},
|
||||
entity.GoogleIndexConfigKeyCheckInterval: {Key: entity.GoogleIndexConfigKeyCheckInterval, Value: "60", Type: entity.ConfigTypeInt},
|
||||
entity.GoogleIndexConfigKeyBatchSize: {Key: entity.GoogleIndexConfigKeyBatchSize, Value: "10", Type: entity.ConfigTypeInt},
|
||||
entity.GoogleIndexConfigKeyConcurrency: {Key: entity.GoogleIndexConfigKeyConcurrency, Value: "2", Type: entity.ConfigTypeInt},
|
||||
entity.GoogleIndexConfigKeyRetryAttempts: {Key: entity.GoogleIndexConfigKeyRetryAttempts, Value: "3", Type: entity.ConfigTypeInt},
|
||||
entity.GoogleIndexConfigKeyRetryDelay: {Key: entity.GoogleIndexConfigKeyRetryDelay, Value: "2", Type: entity.ConfigTypeInt},
|
||||
entity.GoogleIndexConfigKeyAutoSitemap: {Key: entity.GoogleIndexConfigKeyAutoSitemap, Value: "false", Type: entity.ConfigTypeBool},
|
||||
entity.GoogleIndexConfigKeySitemapPath: {Key: entity.GoogleIndexConfigKeySitemapPath, Value: "/sitemap.xml", Type: entity.ConfigTypeString},
|
||||
}
|
||||
|
||||
// 检查现有配置中是否有缺失的配置项
|
||||
@@ -304,6 +324,9 @@ func (r *SystemConfigRepositoryImpl) ValidateConfigIntegrity() error {
|
||||
entity.ConfigKeyMaintenanceMode,
|
||||
entity.ConfigKeyEnableRegister,
|
||||
entity.ConfigKeyThirdPartyStatsCode,
|
||||
// Google索引配置
|
||||
entity.GoogleIndexConfigKeyEnabled,
|
||||
entity.GoogleIndexConfigKeySiteName,
|
||||
}
|
||||
|
||||
existingKeys := make(map[string]bool)
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
type TaskItemRepository interface {
|
||||
GetByID(id uint) (*entity.TaskItem, error)
|
||||
Create(item *entity.TaskItem) error
|
||||
Update(item *entity.TaskItem) error
|
||||
Delete(id uint) error
|
||||
DeleteByTaskID(taskID uint) error
|
||||
GetByTaskIDAndStatus(taskID uint, status string) ([]*entity.TaskItem, error)
|
||||
@@ -19,7 +20,25 @@ type TaskItemRepository interface {
|
||||
UpdateStatus(id uint, status string) error
|
||||
UpdateStatusAndOutput(id uint, status, outputData string) error
|
||||
GetStatsByTaskID(taskID uint) (map[string]int, error)
|
||||
GetIndexStats() (map[string]int, error)
|
||||
ResetProcessingItems(taskID uint) error
|
||||
|
||||
// Google索引专用方法
|
||||
GetDistinctProcessedURLs() ([]string, error)
|
||||
GetLatestURLStatus(url string) (*entity.TaskItem, error)
|
||||
UpsertURLStatusRecords(taskID uint, urlResults []*URLStatusResult) error
|
||||
CleanupOldRecords() error
|
||||
}
|
||||
|
||||
// URLStatusResult 用于批量处理的结果
|
||||
type URLStatusResult struct {
|
||||
URL string
|
||||
IndexStatus string
|
||||
InspectResult string
|
||||
MobileFriendly bool
|
||||
StatusCode int
|
||||
LastCrawled *time.Time
|
||||
ErrorMessage string
|
||||
}
|
||||
|
||||
// TaskItemRepositoryImpl 任务项仓库实现
|
||||
@@ -49,6 +68,33 @@ func (r *TaskItemRepositoryImpl) Create(item *entity.TaskItem) error {
|
||||
return r.db.Create(item).Error
|
||||
}
|
||||
|
||||
// Update 更新任务项
|
||||
func (r *TaskItemRepositoryImpl) Update(item *entity.TaskItem) error {
|
||||
startTime := utils.GetCurrentTime()
|
||||
err := r.db.Model(&entity.TaskItem{}).Where("id = ?", item.ID).Updates(map[string]interface{}{
|
||||
"status": item.Status,
|
||||
"error_message": item.ErrorMessage,
|
||||
"index_status": item.IndexStatus,
|
||||
"mobile_friendly": item.MobileFriendly,
|
||||
"last_crawled": item.LastCrawled,
|
||||
"status_code": item.StatusCode,
|
||||
"input_data": item.InputData,
|
||||
"output_data": item.OutputData,
|
||||
"process_log": item.ProcessLog,
|
||||
"url": item.URL,
|
||||
"inspect_result": item.InspectResult,
|
||||
"processed_at": item.ProcessedAt,
|
||||
"updated_at": time.Now(),
|
||||
}).Error
|
||||
updateDuration := time.Since(startTime)
|
||||
if err != nil {
|
||||
utils.Error("Update任务项失败: ID=%d, 错误=%v, 更新耗时=%v", item.ID, err, updateDuration)
|
||||
return err
|
||||
}
|
||||
utils.Debug("Update任务项成功: ID=%d, 更新耗时=%v", item.ID, updateDuration)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete 删除任务项
|
||||
func (r *TaskItemRepositoryImpl) Delete(id uint) error {
|
||||
return r.db.Delete(&entity.TaskItem{}, id).Error
|
||||
@@ -182,3 +228,180 @@ func (r *TaskItemRepositoryImpl) ResetProcessingItems(taskID uint) error {
|
||||
utils.Debug("ResetProcessingItems成功: 任务ID=%d, 更新耗时=%v", taskID, updateDuration)
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetIndexStats 获取索引统计信息
|
||||
func (r *TaskItemRepositoryImpl) GetIndexStats() (map[string]int, error) {
|
||||
stats := make(map[string]int)
|
||||
|
||||
// 统计各种状态的数量
|
||||
statuses := []string{"completed", "failed", "pending"}
|
||||
|
||||
for _, status := range statuses {
|
||||
var count int64
|
||||
err := r.db.Model(&entity.TaskItem{}).Where("status = ?", status).Count(&count).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
switch status {
|
||||
case "completed":
|
||||
stats["indexed"] = int(count)
|
||||
case "failed":
|
||||
stats["error"] = int(count)
|
||||
case "pending":
|
||||
stats["not_indexed"] = int(count)
|
||||
}
|
||||
}
|
||||
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
// GetDistinctProcessedURLs 获取所有已处理的URL(去重)
|
||||
func (r *TaskItemRepositoryImpl) GetDistinctProcessedURLs() ([]string, error) {
|
||||
startTime := utils.GetCurrentTime()
|
||||
var urls []string
|
||||
|
||||
// 只返回成功处理的URL,避免处理失败的URL重复尝试
|
||||
err := r.db.Model(&entity.TaskItem{}).
|
||||
Where("status = ? AND url != ?", "completed", "").
|
||||
Distinct("url").
|
||||
Pluck("url", &urls).Error
|
||||
|
||||
queryDuration := time.Since(startTime)
|
||||
if err != nil {
|
||||
utils.Error("GetDistinctProcessedURLs失败: 错误=%v, 查询耗时=%v", err, queryDuration)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
utils.Debug("GetDistinctProcessedURLs成功: URL数量=%d, 查询耗时=%v", len(urls), queryDuration)
|
||||
return urls, nil
|
||||
}
|
||||
|
||||
// GetLatestURLStatus 获取URL的最新处理状态
|
||||
func (r *TaskItemRepositoryImpl) GetLatestURLStatus(url string) (*entity.TaskItem, error) {
|
||||
startTime := utils.GetCurrentTime()
|
||||
var item entity.TaskItem
|
||||
|
||||
err := r.db.Where("url = ?", url).
|
||||
Order("created_at DESC").
|
||||
First(&item).Error
|
||||
|
||||
queryDuration := time.Since(startTime)
|
||||
if err != nil {
|
||||
if err == gorm.ErrRecordNotFound {
|
||||
utils.Debug("GetLatestURLStatus: URL未找到=%s, 查询耗时=%v", url, queryDuration)
|
||||
return nil, nil
|
||||
}
|
||||
utils.Error("GetLatestURLStatus失败: URL=%s, 错误=%v, 查询耗时=%v", url, err, queryDuration)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
utils.Debug("GetLatestURLStatus成功: URL=%s, 状态=%s, 查询耗时=%v", url, item.Status, queryDuration)
|
||||
return &item, nil
|
||||
}
|
||||
|
||||
// UpsertURLStatusRecords 批量创建或更新URL状态
|
||||
func (r *TaskItemRepositoryImpl) UpsertURLStatusRecords(taskID uint, urlResults []*URLStatusResult) error {
|
||||
startTime := utils.GetCurrentTime()
|
||||
|
||||
if len(urlResults) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 批量操作,减少数据库查询次数
|
||||
for _, result := range urlResults {
|
||||
// 查找现有记录
|
||||
existing, err := r.GetLatestURLStatus(result.URL)
|
||||
|
||||
if err != nil && err != gorm.ErrRecordNotFound {
|
||||
utils.Error("UpsertURLStatusRecords查询失败: URL=%s, 错误=%v", result.URL, err)
|
||||
continue
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
|
||||
if existing != nil && existing.ID > 0 {
|
||||
// 更新现有记录(只更新状态变化的)
|
||||
if existing.IndexStatus != result.IndexStatus || existing.StatusCode != result.StatusCode {
|
||||
existing.IndexStatus = result.IndexStatus
|
||||
existing.InspectResult = result.InspectResult
|
||||
existing.MobileFriendly = result.MobileFriendly
|
||||
existing.StatusCode = result.StatusCode
|
||||
existing.LastCrawled = result.LastCrawled
|
||||
existing.ErrorMessage = result.ErrorMessage
|
||||
existing.ProcessedAt = &now
|
||||
|
||||
if err := r.Update(existing); err != nil {
|
||||
utils.Error("UpsertURLStatusRecords更新失败: URL=%s, 错误=%v", result.URL, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// 创建新记录
|
||||
newItem := &entity.TaskItem{
|
||||
TaskID: taskID,
|
||||
URL: result.URL,
|
||||
Status: "completed",
|
||||
IndexStatus: result.IndexStatus,
|
||||
InspectResult: result.InspectResult,
|
||||
MobileFriendly: result.MobileFriendly,
|
||||
StatusCode: result.StatusCode,
|
||||
LastCrawled: result.LastCrawled,
|
||||
ErrorMessage: result.ErrorMessage,
|
||||
ProcessedAt: &now,
|
||||
}
|
||||
|
||||
if err := r.Create(newItem); err != nil {
|
||||
utils.Error("UpsertURLStatusRecords创建失败: URL=%s, 错误=%v", result.URL, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
totalDuration := time.Since(startTime)
|
||||
utils.Info("UpsertURLStatusRecords完成: 数量=%d, 耗时=%v", len(urlResults), totalDuration)
|
||||
return nil
|
||||
}
|
||||
|
||||
// CleanupOldRecords 清理旧记录,保留每个URL的最新记录
|
||||
func (r *TaskItemRepositoryImpl) CleanupOldRecords() error {
|
||||
startTime := utils.GetCurrentTime()
|
||||
|
||||
// 1. 找出每个URL的最新记录ID
|
||||
var latestIDs []uint
|
||||
err := r.db.Table("task_items").
|
||||
Select("MAX(id) as id").
|
||||
Where("url != '' AND status = ?", "completed").
|
||||
Group("url").
|
||||
Pluck("id", &latestIDs).Error
|
||||
|
||||
if err != nil {
|
||||
utils.Error("CleanupOldRecords获取最新ID失败: 错误=%v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
// 2. 删除所有非最新的已完成记录
|
||||
deleteResult := r.db.Where("status = ? AND id NOT IN (?)", "completed", latestIDs).
|
||||
Delete(&entity.TaskItem{})
|
||||
|
||||
if deleteResult.Error != nil {
|
||||
utils.Error("CleanupOldRecords删除旧记录失败: 错误=%v", deleteResult.Error)
|
||||
return deleteResult.Error
|
||||
}
|
||||
|
||||
// 3. 清理失败的旧记录(保留1周)
|
||||
failureCutoff := time.Now().AddDate(0, 0, -7)
|
||||
failureDeleteResult := r.db.Where("status = ? AND created_at < ?", "failed", failureCutoff).
|
||||
Delete(&entity.TaskItem{})
|
||||
|
||||
if failureDeleteResult.Error != nil {
|
||||
utils.Error("CleanupOldRecords删除失败记录失败: 错误=%v", failureDeleteResult.Error)
|
||||
return failureDeleteResult.Error
|
||||
}
|
||||
|
||||
totalDuration := time.Since(startTime)
|
||||
utils.Info("CleanupOldRecords完成: 删除完成记录=%d, 删除失败记录=%d, 耗时=%v",
|
||||
deleteResult.RowsAffected, failureDeleteResult.RowsAffected, totalDuration)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ type TaskRepository interface {
|
||||
UpdateTaskStats(id uint, processed, success, failed int) error
|
||||
UpdateStartedAt(id uint) error
|
||||
UpdateCompletedAt(id uint) error
|
||||
UpdateTotalItems(id uint, totalItems int) error
|
||||
}
|
||||
|
||||
// TaskRepositoryImpl 任务仓库实现
|
||||
@@ -243,3 +244,16 @@ func (r *TaskRepositoryImpl) UpdateCompletedAt(id uint) error {
|
||||
utils.Debug("UpdateCompletedAt成功: ID=%d, 更新耗时=%v", id, updateDuration)
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateTotalItems 更新任务总项目数
|
||||
func (r *TaskRepositoryImpl) UpdateTotalItems(id uint, totalItems int) error {
|
||||
startTime := utils.GetCurrentTime()
|
||||
err := r.db.Model(&entity.Task{}).Where("id = ?", id).Update("total_items", totalItems).Error
|
||||
updateDuration := time.Since(startTime)
|
||||
if err != nil {
|
||||
utils.Error("UpdateTotalItems失败: ID=%d, 总项目数=%d, 错误=%v, 更新耗时=%v", id, totalItems, err, updateDuration)
|
||||
return err
|
||||
}
|
||||
utils.Debug("UpdateTotalItems成功: ID=%d, 总项目数=%d, 更新耗时=%v", id, totalItems, updateDuration)
|
||||
return nil
|
||||
}
|
||||
|
||||
20
go.mod
20
go.mod
@@ -15,19 +15,32 @@ require (
|
||||
github.com/robfig/cron/v3 v3.0.1
|
||||
github.com/silenceper/wechat/v2 v2.1.10
|
||||
golang.org/x/crypto v0.41.0
|
||||
golang.org/x/oauth2 v0.30.0
|
||||
google.golang.org/api v0.197.0
|
||||
gorm.io/driver/postgres v1.6.0
|
||||
gorm.io/gorm v1.30.0
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go/auth v0.9.3 // indirect
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.5.0 // indirect
|
||||
github.com/andybalholm/brotli v1.1.1 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/bradfitz/gomemcache v0.0.0-20220106215444-fb4bf637b56d // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/go-logr/logr v1.4.2 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-redis/redis/v8 v8.11.5 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
|
||||
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/google/s2a-go v0.1.8 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.13.0 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/prometheus/client_model v0.6.2 // indirect
|
||||
github.com/prometheus/common v0.66.1 // indirect
|
||||
@@ -36,8 +49,15 @@ require (
|
||||
github.com/tidwall/gjson v1.14.1 // indirect
|
||||
github.com/tidwall/match v1.1.1 // indirect
|
||||
github.com/tidwall/pretty v1.2.0 // indirect
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 // indirect
|
||||
go.opentelemetry.io/otel v1.29.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.29.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.29.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
||||
golang.org/x/image v0.32.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect
|
||||
google.golang.org/grpc v1.66.1 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
|
||||
95
go.sum
95
go.sum
@@ -1,3 +1,11 @@
|
||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
cloud.google.com/go/auth v0.9.3 h1:VOEUIAADkkLtyfr3BLa3R8Ed/j6w1jTBmARx+wb5w5U=
|
||||
cloud.google.com/go/auth v0.9.3/go.mod h1:7z6VY+7h3KUdRov5F1i8NDP5ZzWKYmEPO842BgCsmTk=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc=
|
||||
cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY=
|
||||
cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk=
|
||||
github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
|
||||
github.com/alicebob/miniredis/v2 v2.30.0 h1:uA3uhDbCxfO9+DI/DuGeAMr9qI+noVWwGPNTFuKID5M=
|
||||
@@ -13,22 +21,31 @@ github.com/bytedance/sonic v1.13.3/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1
|
||||
github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
|
||||
github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA=
|
||||
github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
|
||||
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
|
||||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4=
|
||||
github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
|
||||
github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
|
||||
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/fogleman/gg v1.3.0 h1:/7zJX8F6AaYQc57WQCyN9cAIz+4bCJGO9B+dyW29am8=
|
||||
github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
@@ -44,6 +61,11 @@ github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fq
|
||||
github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk=
|
||||
github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ=
|
||||
github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
|
||||
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
|
||||
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
|
||||
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
|
||||
@@ -72,23 +94,43 @@ github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1
|
||||
github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
|
||||
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g=
|
||||
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
|
||||
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
|
||||
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
|
||||
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
|
||||
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
|
||||
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
|
||||
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
|
||||
github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM=
|
||||
github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA=
|
||||
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA=
|
||||
github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s=
|
||||
github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A=
|
||||
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=
|
||||
github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
@@ -166,6 +208,7 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
|
||||
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
|
||||
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
|
||||
github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
|
||||
@@ -213,6 +256,16 @@ github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3i
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64 h1:5mLPGnFdSsevFRFc9q3yYbBkB6tsm4aCwwQV/j1JQAQ=
|
||||
github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
|
||||
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
|
||||
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8=
|
||||
go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw=
|
||||
go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8=
|
||||
go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc=
|
||||
go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8=
|
||||
go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4=
|
||||
go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
|
||||
@@ -226,24 +279,38 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y
|
||||
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4=
|
||||
golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/image v0.32.0 h1:6lZQWq75h7L5IWNk0r+SCpUJ6tUVd3v4ZHnbRKLkUDQ=
|
||||
golang.org/x/image v0.32.0/go.mod h1:/R37rrQmKXtO6tYXAjtDLwQgFLHmhW+V6ayXlxzP2Pc=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE=
|
||||
golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI=
|
||||
golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
|
||||
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
@@ -274,18 +341,44 @@ golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=
|
||||
golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U=
|
||||
golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/api v0.197.0 h1:x6CwqQLsFiA5JKAiGyGBjc2bNtHtLddhJCE2IKuhhcQ=
|
||||
google.golang.org/api v0.197.0/go.mod h1:AuOuo20GoQ331nq7DquGHlU6d+2wN2fZ8O0ta60nRNw=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
|
||||
google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 h1:BulPr26Jqjnd4eYDVe+YvyR7Yc2vJGkO5/0UxD0/jZU=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d h1:kHjw/5UfflP/L5EbledDrcG4C2597RtymmGRZvHiCuY=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d/go.mod h1:mw8MG/Qz5wfgYr6VqVCiZcHe/GJEfI+oGGDCohaVgB0=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
|
||||
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
|
||||
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
|
||||
google.golang.org/grpc v1.66.1 h1:hO5qAXR19+/Z44hmvIM4dQFMSYX9XcWsByfoxutBpAM=
|
||||
google.golang.org/grpc v1.66.1/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
|
||||
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
|
||||
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
|
||||
@@ -314,4 +407,6 @@ gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4=
|
||||
gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo=
|
||||
gorm.io/gorm v1.30.0 h1:qbT5aPv1UH8gI99OsRlvDToLxW5zR7FzS9acZDOZcgs=
|
||||
gorm.io/gorm v1.30.0/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
|
||||
|
||||
1030
handlers/google_index_handler.go
Normal file
1030
handlers/google_index_handler.go
Normal file
File diff suppressed because it is too large
Load Diff
@@ -452,9 +452,16 @@ func (h *PublicAPIHandler) recordAPIAccessToDB(ip, userAgent, endpoint, method s
|
||||
requestParams interface{}, responseStatus int, responseData interface{},
|
||||
processCount int, errorMessage string, processingTime int64) {
|
||||
|
||||
// 只记录重要的API访问(有错误或处理时间较长的)
|
||||
if errorMessage == "" && processingTime < 1000 && responseStatus < 400 {
|
||||
return // 跳过正常的快速请求
|
||||
// 判断是否为关键端点,需要强制记录日志
|
||||
isKeyEndpoint := strings.Contains(endpoint, "/api/public/resources/batch-add") ||
|
||||
strings.Contains(endpoint, "/api/admin/") ||
|
||||
strings.Contains(endpoint, "/telegram/webhook") ||
|
||||
strings.Contains(endpoint, "/api/public/resources/search") ||
|
||||
strings.Contains(endpoint, "/api/public/hot-drama")
|
||||
|
||||
// 只记录重要的API访问(有错误或处理时间较长的)或者是关键端点
|
||||
if errorMessage == "" && processingTime < 1000 && responseStatus < 400 && !isKeyEndpoint {
|
||||
return // 跳过正常的快速请求,但记录关键端点
|
||||
}
|
||||
|
||||
// 转换参数为JSON字符串
|
||||
@@ -492,3 +499,36 @@ func (h *PublicAPIHandler) recordAPIAccessToDB(ip, userAgent, endpoint, method s
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// GetPublicSiteVerificationCode 获取网站验证代码(公开访问)
|
||||
func GetPublicSiteVerificationCode(c *gin.Context) {
|
||||
// 获取站点URL配置
|
||||
siteURL, err := repoManager.SystemConfigRepository.GetConfigValue("site_url")
|
||||
if err != nil || siteURL == "" {
|
||||
c.JSON(400, gin.H{
|
||||
"success": false,
|
||||
"message": "站点URL未配置",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// 生成Google Search Console验证代码示例
|
||||
verificationCode := map[string]interface{}{
|
||||
"site_url": siteURL,
|
||||
"verification_methods": map[string]string{
|
||||
"html_tag": `<meta name="google-site-verification" content="your-verification-code">`,
|
||||
"dns_txt": `google-site-verification=your-verification-code`,
|
||||
"html_file": `google1234567890abcdef.html`,
|
||||
},
|
||||
"instructions": map[string]string{
|
||||
"html_tag": "请将以下meta标签添加到您网站的首页<head>部分中",
|
||||
"dns_txt": "请添加以下TXT记录到您的DNS配置中",
|
||||
"html_file": "请在网站根目录创建包含指定内容的HTML文件",
|
||||
},
|
||||
}
|
||||
|
||||
c.JSON(200, gin.H{
|
||||
"success": true,
|
||||
"data": verificationCode,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -18,11 +18,15 @@ func GetSchedulerStatus(c *gin.Context) {
|
||||
repoManager.CksRepository,
|
||||
repoManager.TagRepository,
|
||||
repoManager.CategoryRepository,
|
||||
repoManager.TaskItemRepository,
|
||||
repoManager.TaskRepository,
|
||||
)
|
||||
|
||||
status := gin.H{
|
||||
"hot_drama_scheduler_running": scheduler.IsHotDramaSchedulerRunning(),
|
||||
"ready_resource_scheduler_running": scheduler.IsReadyResourceRunning(),
|
||||
"google_index_scheduler_running": scheduler.IsGoogleIndexSchedulerRunning(),
|
||||
"sitemap_scheduler_running": scheduler.IsSitemapSchedulerRunning(),
|
||||
}
|
||||
|
||||
SuccessResponse(c, status)
|
||||
@@ -39,6 +43,8 @@ func StartHotDramaScheduler(c *gin.Context) {
|
||||
repoManager.CksRepository,
|
||||
repoManager.TagRepository,
|
||||
repoManager.CategoryRepository,
|
||||
repoManager.TaskItemRepository,
|
||||
repoManager.TaskRepository,
|
||||
)
|
||||
if scheduler.IsHotDramaSchedulerRunning() {
|
||||
ErrorResponse(c, "热播剧定时任务已在运行中", http.StatusBadRequest)
|
||||
@@ -59,6 +65,8 @@ func StopHotDramaScheduler(c *gin.Context) {
|
||||
repoManager.CksRepository,
|
||||
repoManager.TagRepository,
|
||||
repoManager.CategoryRepository,
|
||||
repoManager.TaskItemRepository,
|
||||
repoManager.TaskRepository,
|
||||
)
|
||||
if !scheduler.IsHotDramaSchedulerRunning() {
|
||||
ErrorResponse(c, "热播剧定时任务未在运行", http.StatusBadRequest)
|
||||
@@ -79,6 +87,8 @@ func TriggerHotDramaScheduler(c *gin.Context) {
|
||||
repoManager.CksRepository,
|
||||
repoManager.TagRepository,
|
||||
repoManager.CategoryRepository,
|
||||
repoManager.TaskItemRepository,
|
||||
repoManager.TaskRepository,
|
||||
)
|
||||
scheduler.StartHotDramaScheduler() // 直接启动一次
|
||||
SuccessResponse(c, gin.H{"message": "手动触发热播剧定时任务成功"})
|
||||
@@ -95,6 +105,8 @@ func FetchHotDramaNames(c *gin.Context) {
|
||||
repoManager.CksRepository,
|
||||
repoManager.TagRepository,
|
||||
repoManager.CategoryRepository,
|
||||
repoManager.TaskItemRepository,
|
||||
repoManager.TaskRepository,
|
||||
)
|
||||
names, err := scheduler.GetHotDramaNames()
|
||||
if err != nil {
|
||||
@@ -115,6 +127,8 @@ func StartReadyResourceScheduler(c *gin.Context) {
|
||||
repoManager.CksRepository,
|
||||
repoManager.TagRepository,
|
||||
repoManager.CategoryRepository,
|
||||
repoManager.TaskItemRepository,
|
||||
repoManager.TaskRepository,
|
||||
)
|
||||
if scheduler.IsReadyResourceRunning() {
|
||||
ErrorResponse(c, "待处理资源自动处理任务已在运行中", http.StatusBadRequest)
|
||||
@@ -135,6 +149,8 @@ func StopReadyResourceScheduler(c *gin.Context) {
|
||||
repoManager.CksRepository,
|
||||
repoManager.TagRepository,
|
||||
repoManager.CategoryRepository,
|
||||
repoManager.TaskItemRepository,
|
||||
repoManager.TaskRepository,
|
||||
)
|
||||
if !scheduler.IsReadyResourceRunning() {
|
||||
ErrorResponse(c, "待处理资源自动处理任务未在运行", http.StatusBadRequest)
|
||||
@@ -155,6 +171,8 @@ func TriggerReadyResourceScheduler(c *gin.Context) {
|
||||
repoManager.CksRepository,
|
||||
repoManager.TagRepository,
|
||||
repoManager.CategoryRepository,
|
||||
repoManager.TaskItemRepository,
|
||||
repoManager.TaskRepository,
|
||||
)
|
||||
scheduler.StartReadyResourceScheduler() // 直接启动一次
|
||||
SuccessResponse(c, gin.H{"message": "手动触发待处理资源自动处理任务成功"})
|
||||
|
||||
@@ -90,6 +90,8 @@ func GetSitemapConfig(c *gin.Context) {
|
||||
enabled, err := scheduler.GetGlobalScheduler(
|
||||
hotDramaRepo, readyResourceRepo, resourceRepo, systemConfigRepo,
|
||||
panRepo, cksRepo, tagRepo, categoryRepo,
|
||||
repoManager.TaskItemRepository,
|
||||
repoManager.TaskRepository,
|
||||
).GetSitemapConfig()
|
||||
if err != nil && err != gorm.ErrRecordNotFound {
|
||||
// 如果获取失败,尝试从配置表中获取
|
||||
@@ -140,6 +142,8 @@ func UpdateSitemapConfig(c *gin.Context) {
|
||||
if err := scheduler.GetGlobalScheduler(
|
||||
hotDramaRepo, readyResourceRepo, resourceRepo, systemConfigRepo,
|
||||
panRepo, cksRepo, tagRepo, categoryRepo,
|
||||
repoManager.TaskItemRepository,
|
||||
repoManager.TaskRepository,
|
||||
).UpdateSitemapConfig(config.AutoGenerate); err != nil {
|
||||
ErrorResponse(c, "更新调度器配置失败", http.StatusInternalServerError)
|
||||
return
|
||||
@@ -175,11 +179,15 @@ func UpdateSitemapConfig(c *gin.Context) {
|
||||
scheduler.GetGlobalScheduler(
|
||||
hotDramaRepo, readyResourceRepo, resourceRepo, systemConfigRepo,
|
||||
panRepo, cksRepo, tagRepo, categoryRepo,
|
||||
repoManager.TaskItemRepository,
|
||||
repoManager.TaskRepository,
|
||||
).StartSitemapScheduler()
|
||||
} else {
|
||||
scheduler.GetGlobalScheduler(
|
||||
hotDramaRepo, readyResourceRepo, resourceRepo, systemConfigRepo,
|
||||
panRepo, cksRepo, tagRepo, categoryRepo,
|
||||
repoManager.TaskItemRepository,
|
||||
repoManager.TaskRepository,
|
||||
).StopSitemapScheduler()
|
||||
}
|
||||
|
||||
@@ -201,6 +209,8 @@ func GenerateSitemap(c *gin.Context) {
|
||||
globalScheduler := scheduler.GetGlobalScheduler(
|
||||
hotDramaRepo, readyResourceRepo, resourceRepo, systemConfigRepo,
|
||||
panRepo, cksRepo, tagRepo, categoryRepo,
|
||||
repoManager.TaskItemRepository,
|
||||
repoManager.TaskRepository,
|
||||
)
|
||||
|
||||
// 手动触发sitemap生成
|
||||
@@ -257,12 +267,16 @@ func GetSitemapStatus(c *gin.Context) {
|
||||
isRunning := scheduler.GetGlobalScheduler(
|
||||
hotDramaRepo, readyResourceRepo, resourceRepo, systemConfigRepo,
|
||||
panRepo, cksRepo, tagRepo, categoryRepo,
|
||||
repoManager.TaskItemRepository,
|
||||
repoManager.TaskRepository,
|
||||
).IsSitemapSchedulerRunning()
|
||||
|
||||
// 获取自动生成功能状态
|
||||
autoGenerateEnabled, err := scheduler.GetGlobalScheduler(
|
||||
hotDramaRepo, readyResourceRepo, resourceRepo, systemConfigRepo,
|
||||
panRepo, cksRepo, tagRepo, categoryRepo,
|
||||
repoManager.TaskItemRepository,
|
||||
repoManager.TaskRepository,
|
||||
).GetSitemapConfig()
|
||||
if err != nil {
|
||||
// 如果调度器获取失败,从配置中获取
|
||||
@@ -382,6 +396,8 @@ func GenerateFullSitemap(c *gin.Context) {
|
||||
globalScheduler := scheduler.GetGlobalScheduler(
|
||||
hotDramaRepo, readyResourceRepo, resourceRepo, systemConfigRepo,
|
||||
panRepo, cksRepo, tagRepo, categoryRepo,
|
||||
repoManager.TaskItemRepository,
|
||||
repoManager.TaskRepository,
|
||||
)
|
||||
|
||||
// 手动触发sitemap生成
|
||||
|
||||
@@ -240,6 +240,8 @@ func UpdateSystemConfig(c *gin.Context) {
|
||||
repoManager.CksRepository,
|
||||
repoManager.TagRepository,
|
||||
repoManager.CategoryRepository,
|
||||
repoManager.TaskItemRepository,
|
||||
repoManager.TaskRepository,
|
||||
)
|
||||
if scheduler != nil {
|
||||
// 只更新被设置的配置
|
||||
@@ -281,6 +283,7 @@ func GetPublicSystemConfig(c *gin.Context) {
|
||||
SuccessResponse(c, configResponse)
|
||||
}
|
||||
|
||||
|
||||
// 新增:配置监控端点
|
||||
func GetConfigStatus(c *gin.Context) {
|
||||
// 获取配置统计信息
|
||||
@@ -366,6 +369,8 @@ func ToggleAutoProcess(c *gin.Context) {
|
||||
repoManager.CksRepository,
|
||||
repoManager.TagRepository,
|
||||
repoManager.CategoryRepository,
|
||||
repoManager.TaskItemRepository,
|
||||
repoManager.TaskRepository,
|
||||
)
|
||||
if scheduler != nil {
|
||||
// 获取其他配置值
|
||||
|
||||
42
main.go
42
main.go
@@ -191,6 +191,8 @@ func main() {
|
||||
repoManager.CksRepository,
|
||||
repoManager.TagRepository,
|
||||
repoManager.CategoryRepository,
|
||||
repoManager.TaskItemRepository,
|
||||
repoManager.TaskRepository,
|
||||
)
|
||||
|
||||
// 根据系统配置启动相应的调度任务
|
||||
@@ -198,6 +200,7 @@ func main() {
|
||||
autoProcessReadyResources, _ := repoManager.SystemConfigRepository.GetConfigBool(entity.ConfigKeyAutoProcessReadyResources)
|
||||
autoTransferEnabled, _ := repoManager.SystemConfigRepository.GetConfigBool(entity.ConfigKeyAutoTransferEnabled)
|
||||
autoSitemapEnabled, _ := repoManager.SystemConfigRepository.GetConfigBool(entity.ConfigKeySitemapAutoGenerateEnabled)
|
||||
autoGoogleIndexEnabled, _ := repoManager.SystemConfigRepository.GetConfigBool(entity.GoogleIndexConfigKeyEnabled)
|
||||
|
||||
globalScheduler.UpdateSchedulerStatusWithAutoTransfer(
|
||||
autoFetchHotDrama,
|
||||
@@ -213,6 +216,14 @@ func main() {
|
||||
utils.Info("系统配置禁用Sitemap自动生成功能")
|
||||
}
|
||||
|
||||
// 根据系统配置启动Google索引调度器
|
||||
if autoGoogleIndexEnabled {
|
||||
globalScheduler.StartGoogleIndexScheduler()
|
||||
utils.Info("系统配置启用Google索引自动提交功能,启动定时任务")
|
||||
} else {
|
||||
utils.Info("系统配置禁用Google索引自动提交功能")
|
||||
}
|
||||
|
||||
utils.Info("调度器初始化完成")
|
||||
|
||||
// 设置公开API中间件的Repository管理器
|
||||
@@ -237,6 +248,17 @@ func main() {
|
||||
reportHandler := handlers.NewReportHandler(repoManager.ReportRepository, repoManager.ResourceRepository)
|
||||
copyrightClaimHandler := handlers.NewCopyrightClaimHandler(repoManager.CopyrightClaimRepository, repoManager.ResourceRepository)
|
||||
|
||||
// 创建Google索引任务处理器
|
||||
googleIndexProcessor := task.NewGoogleIndexProcessor(repoManager)
|
||||
|
||||
// 创建Google索引处理器
|
||||
googleIndexHandler := handlers.NewGoogleIndexHandler(repoManager, taskManager)
|
||||
|
||||
// 注册Google索引处理器到任务管理器
|
||||
taskManager.RegisterProcessor(googleIndexProcessor)
|
||||
|
||||
utils.Info("Google索引功能已启用,注册到任务管理器")
|
||||
|
||||
// API路由
|
||||
api := r.Group("/api")
|
||||
{
|
||||
@@ -363,6 +385,7 @@ func main() {
|
||||
api.GET("/system/config/status", middleware.AuthMiddleware(), middleware.AdminMiddleware(), handlers.GetConfigStatus)
|
||||
api.POST("/system/config/toggle-auto-process", middleware.AuthMiddleware(), middleware.AdminMiddleware(), handlers.ToggleAutoProcess)
|
||||
api.GET("/public/system-config", handlers.GetPublicSystemConfig)
|
||||
api.GET("/public/site-verification", handlers.GetPublicSiteVerificationCode) // 网站验证代码(公开访问)
|
||||
|
||||
// 热播剧管理路由(查询接口无需认证)
|
||||
api.GET("/hot-dramas", handlers.GetHotDramaList)
|
||||
@@ -521,6 +544,22 @@ func main() {
|
||||
api.POST("/sitemap/generate", middleware.AuthMiddleware(), middleware.AdminMiddleware(), handlers.GenerateSitemap)
|
||||
api.GET("/sitemap/status", middleware.AuthMiddleware(), middleware.AdminMiddleware(), handlers.GetSitemapStatus)
|
||||
api.POST("/sitemap/full-generate", middleware.AuthMiddleware(), middleware.AdminMiddleware(), handlers.GenerateFullSitemap)
|
||||
|
||||
// Google索引管理API
|
||||
api.GET("/google-index/config", middleware.AuthMiddleware(), middleware.AdminMiddleware(), googleIndexHandler.GetConfig)
|
||||
api.GET("/google-index/config-all", middleware.AuthMiddleware(), middleware.AdminMiddleware(), googleIndexHandler.GetAllConfig) // 获取所有配置
|
||||
api.POST("/google-index/config", middleware.AuthMiddleware(), middleware.AdminMiddleware(), googleIndexHandler.UpdateConfig)
|
||||
api.POST("/google-index/config/update", middleware.AuthMiddleware(), middleware.AdminMiddleware(), googleIndexHandler.UpdateGoogleIndexConfig) // 分组配置更新
|
||||
api.GET("/google-index/status", middleware.AuthMiddleware(), middleware.AdminMiddleware(), googleIndexHandler.GetStatus) // 获取状态
|
||||
api.POST("/google-index/tasks", middleware.AuthMiddleware(), middleware.AdminMiddleware(), googleIndexHandler.CreateTask)
|
||||
api.GET("/google-index/tasks", middleware.AuthMiddleware(), middleware.AdminMiddleware(), googleIndexHandler.GetTasks)
|
||||
api.GET("/google-index/tasks/:id", middleware.AuthMiddleware(), middleware.AdminMiddleware(), googleIndexHandler.GetTaskStatus)
|
||||
api.POST("/google-index/tasks/:id/start", middleware.AuthMiddleware(), middleware.AdminMiddleware(), googleIndexHandler.StartTask)
|
||||
api.GET("/google-index/tasks/:id/items", middleware.AuthMiddleware(), middleware.AdminMiddleware(), googleIndexHandler.GetTaskItems)
|
||||
|
||||
// Google索引凭据上传和验证API
|
||||
api.POST("/google-index/upload-credentials", middleware.AuthMiddleware(), middleware.AdminMiddleware(), googleIndexHandler.UploadCredentials)
|
||||
api.POST("/google-index/validate-credentials", middleware.AuthMiddleware(), middleware.AdminMiddleware(), googleIndexHandler.ValidateCredentials)
|
||||
}
|
||||
|
||||
// 设置监控系统
|
||||
@@ -538,10 +577,11 @@ func main() {
|
||||
|
||||
// 静态文件服务
|
||||
r.Static("/uploads", "./uploads")
|
||||
r.Static("/data", "./data")
|
||||
|
||||
// 添加CORS头到静态文件
|
||||
r.Use(func(c *gin.Context) {
|
||||
if strings.HasPrefix(c.Request.URL.Path, "/uploads/") {
|
||||
if strings.HasPrefix(c.Request.URL.Path, "/uploads/") || strings.HasPrefix(c.Request.URL.Path, "/data/") {
|
||||
c.Header("Access-Control-Allow-Origin", "*")
|
||||
c.Header("Access-Control-Allow-Methods", "GET, OPTIONS")
|
||||
c.Header("Access-Control-Allow-Headers", "Origin, Content-Type, Accept")
|
||||
|
||||
@@ -32,8 +32,8 @@ server {
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
}
|
||||
|
||||
# API 路由 - 所有 /api/ 开头的请求转发到后端
|
||||
|
||||
# API 路由 - 所有 /api/ 开头的请求转发到后端
|
||||
location /api/ {
|
||||
proxy_pass http://backend;
|
||||
proxy_set_header Host $host;
|
||||
@@ -55,6 +55,23 @@ server {
|
||||
proxy_intercept_errors on;
|
||||
error_page 502 503 504 /50x.html;
|
||||
}
|
||||
|
||||
|
||||
location ^~ /uploads/ {
|
||||
proxy_pass http://backend/uploads/;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
expires 1y;
|
||||
add_header Cache-Control "public, immutable";
|
||||
|
||||
add_header Access-Control-Allow-Origin "*";
|
||||
add_header Access-Control-Allow-Methods "GET, OPTIONS";
|
||||
add_header Access-Control-Allow-Headers "Origin, Content-Type, Accept";
|
||||
}
|
||||
|
||||
|
||||
# 静态文件路由 - 直接转发到前端
|
||||
location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2|ttf|eot)$ {
|
||||
@@ -69,24 +86,6 @@ server {
|
||||
add_header Cache-Control "public, immutable";
|
||||
}
|
||||
|
||||
# 上传文件路由 - 直接访问后端的上传目录
|
||||
location /uploads/ {
|
||||
proxy_pass http://backend;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
# 缓存设置
|
||||
expires 1y;
|
||||
add_header Cache-Control "public, immutable";
|
||||
|
||||
# 允许跨域访问
|
||||
add_header Access-Control-Allow-Origin "*";
|
||||
add_header Access-Control-Allow-Methods "GET, OPTIONS";
|
||||
add_header Access-Control-Allow-Headers "Origin, Content-Type, Accept";
|
||||
}
|
||||
|
||||
# 微信公众号验证文件路由 - 根目录的TXT文件直接访问后端uploads目录
|
||||
location ~ ^/[^/]+\.txt$ {
|
||||
# 检查文件是否存在于uploads目录
|
||||
|
||||
263
pkg/google/client.go
Normal file
263
pkg/google/client.go
Normal file
@@ -0,0 +1,263 @@
|
||||
package google
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"golang.org/x/oauth2"
|
||||
"golang.org/x/oauth2/google"
|
||||
"google.golang.org/api/option"
|
||||
"google.golang.org/api/searchconsole/v1"
|
||||
)
|
||||
|
||||
// Client Google Search Console API客户端
|
||||
type Client struct {
|
||||
service *searchconsole.Service
|
||||
SiteURL string
|
||||
}
|
||||
|
||||
// Config 配置信息
|
||||
type Config struct {
|
||||
CredentialsFile string `json:"credentials_file"`
|
||||
SiteURL string `json:"site_url"`
|
||||
TokenFile string `json:"token_file"`
|
||||
}
|
||||
|
||||
// URLInspectionRequest URL检查请求
|
||||
type URLInspectionRequest struct {
|
||||
InspectionURL string `json:"inspectionUrl"`
|
||||
SiteURL string `json:"siteUrl"`
|
||||
LanguageCode string `json:"languageCode"`
|
||||
}
|
||||
|
||||
// URLInspectionResult URL检查结果
|
||||
type URLInspectionResult struct {
|
||||
IndexStatusResult struct {
|
||||
IndexingState string `json:"indexingState"`
|
||||
LastCrawled string `json:"lastCrawled"`
|
||||
CrawlErrors []struct {
|
||||
ErrorCode string `json:"errorCode"`
|
||||
} `json:"crawlErrors"`
|
||||
} `json:"indexStatusResult"`
|
||||
MobileUsabilityResult struct {
|
||||
MobileFriendly bool `json:"mobileFriendly"`
|
||||
} `json:"mobileUsabilityResult"`
|
||||
RichResultsResult struct {
|
||||
Detected struct {
|
||||
Items []struct {
|
||||
RichResultType string `json:"richResultType"`
|
||||
} `json:"items"`
|
||||
} `json:"detected"`
|
||||
} `json:"richResultsResult"`
|
||||
}
|
||||
|
||||
// NewClient 创建新的客户端
|
||||
func NewClient(config *Config) (*Client, error) {
|
||||
ctx := context.Background()
|
||||
|
||||
// 读取认证文件
|
||||
credentials, err := os.ReadFile(config.CredentialsFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("读取认证文件失败: %v", err)
|
||||
}
|
||||
|
||||
// 检查凭据类型
|
||||
var credentialsMap map[string]interface{}
|
||||
if err := json.Unmarshal(credentials, &credentialsMap); err != nil {
|
||||
return nil, fmt.Errorf("解析凭据失败: %v", err)
|
||||
}
|
||||
|
||||
// 根据凭据类型创建不同的配置
|
||||
credType, ok := credentialsMap["type"].(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("未知的凭据类型")
|
||||
}
|
||||
|
||||
var client *http.Client
|
||||
if credType == "service_account" {
|
||||
// 服务账号凭据
|
||||
jwtConfig, err := google.JWTConfigFromJSON(credentials, searchconsole.WebmastersScope)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("创建JWT配置失败: %v", err)
|
||||
}
|
||||
client = jwtConfig.Client(ctx)
|
||||
} else {
|
||||
// OAuth2客户端凭据
|
||||
oauthConfig, err := google.ConfigFromJSON(credentials, searchconsole.WebmastersScope)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("创建OAuth配置失败: %v", err)
|
||||
}
|
||||
|
||||
// 尝试从文件读取token
|
||||
token, err := tokenFromFile(config.TokenFile)
|
||||
if err != nil {
|
||||
// 如果没有token,启动web认证流程
|
||||
token = getTokenFromWeb(oauthConfig)
|
||||
saveToken(config.TokenFile, token)
|
||||
}
|
||||
|
||||
client = oauthConfig.Client(ctx, token)
|
||||
}
|
||||
|
||||
// 创建Search Console服务
|
||||
service, err := searchconsole.NewService(ctx, option.WithHTTPClient(client))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("创建Search Console服务失败: %v", err)
|
||||
}
|
||||
|
||||
return &Client{
|
||||
service: service,
|
||||
SiteURL: config.SiteURL,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// InspectURL 检查URL索引状态
|
||||
func (c *Client) InspectURL(url string) (*URLInspectionResult, error) {
|
||||
request := &searchconsole.InspectUrlIndexRequest{
|
||||
InspectionUrl: url,
|
||||
SiteUrl: c.SiteURL,
|
||||
LanguageCode: "zh-CN",
|
||||
}
|
||||
|
||||
call := c.service.UrlInspection.Index.Inspect(request)
|
||||
response, err := call.Do()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("检查URL失败: %v", err)
|
||||
}
|
||||
|
||||
// 转换响应
|
||||
result := &URLInspectionResult{}
|
||||
if response.InspectionResult != nil {
|
||||
if response.InspectionResult.IndexStatusResult != nil {
|
||||
result.IndexStatusResult.IndexingState = string(response.InspectionResult.IndexStatusResult.IndexingState)
|
||||
if response.InspectionResult.IndexStatusResult.LastCrawlTime != "" {
|
||||
result.IndexStatusResult.LastCrawled = response.InspectionResult.IndexStatusResult.LastCrawlTime
|
||||
}
|
||||
}
|
||||
|
||||
if response.InspectionResult.MobileUsabilityResult != nil {
|
||||
result.MobileUsabilityResult.MobileFriendly = response.InspectionResult.MobileUsabilityResult.Verdict == "MOBILE_USABILITY_VERdict_PASS"
|
||||
}
|
||||
|
||||
if response.InspectionResult.RichResultsResult != nil && response.InspectionResult.RichResultsResult.Verdict != "RICH_RESULTS_VERdict_PASS" {
|
||||
// 如果有富媒体结果检查信息
|
||||
result.RichResultsResult.Detected.Items = append(result.RichResultsResult.Detected.Items, struct {
|
||||
RichResultType string `json:"richResultType"`
|
||||
}{
|
||||
RichResultType: "UNKNOWN",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// SubmitSitemap 提交网站地图
|
||||
func (c *Client) SubmitSitemap(sitemapURL string) error {
|
||||
call := c.service.Sitemaps.Submit(c.SiteURL, sitemapURL)
|
||||
err := call.Do()
|
||||
if err != nil {
|
||||
return fmt.Errorf("提交网站地图失败: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetSites 获取已验证的网站列表
|
||||
func (c *Client) GetSites() ([]*searchconsole.WmxSite, error) {
|
||||
call := c.service.Sites.List()
|
||||
response, err := call.Do()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("获取网站列表失败: %v", err)
|
||||
}
|
||||
|
||||
return response.SiteEntry, nil
|
||||
}
|
||||
|
||||
// GetSearchAnalytics 获取搜索分析数据
|
||||
func (c *Client) GetSearchAnalytics(startDate, endDate string) (*searchconsole.SearchAnalyticsQueryResponse, error) {
|
||||
request := &searchconsole.SearchAnalyticsQueryRequest{
|
||||
StartDate: startDate,
|
||||
EndDate: endDate,
|
||||
Type: "web",
|
||||
}
|
||||
|
||||
call := c.service.Searchanalytics.Query(c.SiteURL, request)
|
||||
response, err := call.Do()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("获取搜索分析数据失败: %v", err)
|
||||
}
|
||||
|
||||
return response, nil
|
||||
}
|
||||
|
||||
// getTokenFromWeb 通过web流程获取token
|
||||
func getTokenFromWeb(config *oauth2.Config) *oauth2.Token {
|
||||
authURL := config.AuthCodeURL("state-token", oauth2.AccessTypeOffline)
|
||||
fmt.Printf("请在浏览器中访问以下URL进行认证:\n%s\n", authURL)
|
||||
fmt.Printf("输入授权代码: ")
|
||||
|
||||
var authCode string
|
||||
if _, err := fmt.Scan(&authCode); err != nil {
|
||||
panic(fmt.Sprintf("读取授权代码失败: %v", err))
|
||||
}
|
||||
|
||||
token, err := config.Exchange(oauth2.NoContext, authCode)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("获取token失败: %v", err))
|
||||
}
|
||||
|
||||
return token
|
||||
}
|
||||
|
||||
// tokenFromFile 从文件读取token
|
||||
func tokenFromFile(file string) (*oauth2.Token, error) {
|
||||
f, err := os.Open(file)
|
||||
defer f.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
token := &oauth2.Token{}
|
||||
err = json.NewDecoder(f).Decode(token)
|
||||
return token, err
|
||||
}
|
||||
|
||||
// saveToken 保存token到文件
|
||||
func saveToken(file string, token *oauth2.Token) {
|
||||
fmt.Printf("保存凭证文件到: %s\n", file)
|
||||
f, err := os.OpenFile(file, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("无法保存凭证文件: %v", err))
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
json.NewEncoder(f).Encode(token)
|
||||
}
|
||||
|
||||
// BatchInspectURL 批量检查URL状态
|
||||
func (c *Client) BatchInspectURL(urls []string, callback func(url string, result *URLInspectionResult, err error)) {
|
||||
semaphore := make(chan struct{}, 5) // 限制并发数
|
||||
|
||||
for _, url := range urls {
|
||||
go func(u string) {
|
||||
semaphore <- struct{}{} // 获取信号量
|
||||
defer func() { <-semaphore }() // 释放信号量
|
||||
|
||||
result, err := c.InspectURL(u)
|
||||
callback(u, result, err)
|
||||
}(url)
|
||||
|
||||
// 避免请求过快
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
|
||||
// 等待所有goroutine完成
|
||||
for i := 0; i < cap(semaphore); i++ {
|
||||
semaphore <- struct{}{}
|
||||
}
|
||||
}
|
||||
166
pkg/google/sitemap.go
Normal file
166
pkg/google/sitemap.go
Normal file
@@ -0,0 +1,166 @@
|
||||
package google
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Sitemap 网站地图结构
|
||||
type Sitemap struct {
|
||||
XMLName xml.Name `xml:"urlset"`
|
||||
Xmlns string `xml:"xmlns,attr"`
|
||||
URLs []SitemapURL `xml:"url"`
|
||||
}
|
||||
|
||||
// SitemapURL 网站地图URL项
|
||||
type SitemapURL struct {
|
||||
Loc string `xml:"loc"`
|
||||
LastMod string `xml:"lastmod,omitempty"`
|
||||
ChangeFreq string `xml:"changefreq,omitempty"`
|
||||
Priority string `xml:"priority,omitempty"`
|
||||
}
|
||||
|
||||
// SitemapIndex 网站地图索引
|
||||
type SitemapIndex struct {
|
||||
XMLName xml.Name `xml:"sitemapindex"`
|
||||
Xmlns string `xml:"xmlns,attr"`
|
||||
Sitemaps []SitemapRef `xml:"sitemap"`
|
||||
}
|
||||
|
||||
// SitemapRef 网站地图引用
|
||||
type SitemapRef struct {
|
||||
Loc string `xml:"loc"`
|
||||
LastMod string `xml:"lastmod,omitempty"`
|
||||
}
|
||||
|
||||
// GenerateSitemap 生成网站地图
|
||||
func GenerateSitemap(urls []string, filename string) error {
|
||||
sitemap := Sitemap{
|
||||
Xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9",
|
||||
}
|
||||
|
||||
for _, url := range urls {
|
||||
sitemapURL := SitemapURL{
|
||||
Loc: strings.TrimSpace(url),
|
||||
LastMod: time.Now().Format("2006-01-02"),
|
||||
ChangeFreq: "weekly",
|
||||
Priority: "0.8",
|
||||
}
|
||||
sitemap.URLs = append(sitemap.URLs, sitemapURL)
|
||||
}
|
||||
|
||||
data, err := xml.MarshalIndent(sitemap, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("生成XML失败: %v", err)
|
||||
}
|
||||
|
||||
// 添加XML头部
|
||||
xmlData := []byte(xml.Header + string(data))
|
||||
|
||||
return os.WriteFile(filename, xmlData, 0644)
|
||||
}
|
||||
|
||||
// GenerateSitemapIndex 生成网站地图索引
|
||||
func GenerateSitemapIndex(sitemaps []string, filename string) error {
|
||||
index := SitemapIndex{
|
||||
Xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9",
|
||||
}
|
||||
|
||||
for _, sitemap := range sitemaps {
|
||||
ref := SitemapRef{
|
||||
Loc: strings.TrimSpace(sitemap),
|
||||
LastMod: time.Now().Format("2006-01-02"),
|
||||
}
|
||||
index.Sitemaps = append(index.Sitemaps, ref)
|
||||
}
|
||||
|
||||
data, err := xml.MarshalIndent(index, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("生成XML失败: %v", err)
|
||||
}
|
||||
|
||||
// 添加XML头部
|
||||
xmlData := []byte(xml.Header + string(data))
|
||||
|
||||
return os.WriteFile(filename, xmlData, 0644)
|
||||
}
|
||||
|
||||
// SplitSitemap 将大量URL分割成多个网站地图
|
||||
func SplitSitemap(urls []string, maxURLsPerSitemap int, baseURL string) ([]string, error) {
|
||||
if maxURLsPerSitemap <= 0 {
|
||||
maxURLsPerSitemap = 50000 // Google限制
|
||||
}
|
||||
|
||||
var sitemapFiles []string
|
||||
totalURLs := len(urls)
|
||||
sitemapCount := (totalURLs + maxURLsPerSitemap - 1) / maxURLsPerSitemap
|
||||
|
||||
for i := 0; i < sitemapCount; i++ {
|
||||
start := i * maxURLsPerSitemap
|
||||
end := start + maxURLsPerSitemap
|
||||
if end > totalURLs {
|
||||
end = totalURLs
|
||||
}
|
||||
|
||||
sitemapURLs := urls[start:end]
|
||||
filename := fmt.Sprintf("sitemap_part_%d.xml", i+1)
|
||||
|
||||
err := GenerateSitemap(sitemapURLs, filename)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("生成网站地图 %s 失败: %v", filename, err)
|
||||
}
|
||||
|
||||
sitemapFiles = append(sitemapFiles, baseURL+filename)
|
||||
fmt.Printf("生成网站地图: %s (%d URLs)\n", filename, len(sitemapURLs))
|
||||
}
|
||||
|
||||
// 生成网站地图索引
|
||||
if len(sitemapFiles) > 1 {
|
||||
indexFiles := make([]string, len(sitemapFiles))
|
||||
for i, file := range sitemapFiles {
|
||||
indexFiles[i] = file
|
||||
}
|
||||
|
||||
err := GenerateSitemapIndex(indexFiles, "sitemap_index.xml")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("生成网站地图索引失败: %v", err)
|
||||
}
|
||||
|
||||
fmt.Printf("生成网站地图索引: sitemap_index.xml\n")
|
||||
}
|
||||
|
||||
return sitemapFiles, nil
|
||||
}
|
||||
|
||||
// PingSearchEngines 通知搜索引擎网站地图更新
|
||||
func PingSearchEngines(sitemapURL string) error {
|
||||
searchEngines := []string{
|
||||
"http://www.google.com/webmasters/sitemaps/ping?sitemap=",
|
||||
"http://www.bing.com/webmaster/ping.aspx?siteMap=",
|
||||
}
|
||||
|
||||
client := &http.Client{Timeout: 10 * time.Second}
|
||||
|
||||
for _, engine := range searchEngines {
|
||||
fullURL := engine + sitemapURL
|
||||
|
||||
resp, err := client.Get(fullURL)
|
||||
if err != nil {
|
||||
fmt.Printf("通知搜索引擎失败 %s: %v\n", engine, err)
|
||||
continue
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
if resp.StatusCode == 200 {
|
||||
fmt.Printf("成功通知: %s\n", engine)
|
||||
} else {
|
||||
fmt.Printf("通知失败: %s (状态码: %d)\n", engine, resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -32,10 +32,10 @@ func GetGlobalMeilisearchManager() *services.MeilisearchManager {
|
||||
}
|
||||
|
||||
// GetGlobalScheduler 获取全局调度器实例(单例模式)
|
||||
func GetGlobalScheduler(hotDramaRepo repo.HotDramaRepository, readyResourceRepo repo.ReadyResourceRepository, resourceRepo repo.ResourceRepository, systemConfigRepo repo.SystemConfigRepository, panRepo repo.PanRepository, cksRepo repo.CksRepository, tagRepo repo.TagRepository, categoryRepo repo.CategoryRepository) *GlobalScheduler {
|
||||
func GetGlobalScheduler(hotDramaRepo repo.HotDramaRepository, readyResourceRepo repo.ReadyResourceRepository, resourceRepo repo.ResourceRepository, systemConfigRepo repo.SystemConfigRepository, panRepo repo.PanRepository, cksRepo repo.CksRepository, tagRepo repo.TagRepository, categoryRepo repo.CategoryRepository, taskItemRepo repo.TaskItemRepository, taskRepo repo.TaskRepository) *GlobalScheduler {
|
||||
once.Do(func() {
|
||||
globalScheduler = &GlobalScheduler{
|
||||
manager: NewManager(hotDramaRepo, readyResourceRepo, resourceRepo, systemConfigRepo, panRepo, cksRepo, tagRepo, categoryRepo),
|
||||
manager: NewManager(hotDramaRepo, readyResourceRepo, resourceRepo, systemConfigRepo, panRepo, cksRepo, tagRepo, categoryRepo, taskItemRepo, taskRepo),
|
||||
}
|
||||
})
|
||||
return globalScheduler
|
||||
@@ -201,3 +201,39 @@ func (gs *GlobalScheduler) TriggerSitemapGeneration() {
|
||||
|
||||
gs.manager.TriggerSitemapGeneration()
|
||||
}
|
||||
|
||||
// StartGoogleIndexScheduler 启动Google索引调度任务
|
||||
func (gs *GlobalScheduler) StartGoogleIndexScheduler() {
|
||||
gs.mutex.Lock()
|
||||
defer gs.mutex.Unlock()
|
||||
|
||||
if gs.manager.IsGoogleIndexRunning() {
|
||||
utils.Debug("Google索引调度任务已在运行中")
|
||||
return
|
||||
}
|
||||
|
||||
gs.manager.StartGoogleIndexScheduler()
|
||||
utils.Info("Google索引调度任务已启动")
|
||||
}
|
||||
|
||||
// StopGoogleIndexScheduler 停止Google索引调度任务
|
||||
func (gs *GlobalScheduler) StopGoogleIndexScheduler() {
|
||||
gs.mutex.Lock()
|
||||
defer gs.mutex.Unlock()
|
||||
|
||||
if !gs.manager.IsGoogleIndexRunning() {
|
||||
utils.Debug("Google索引调度任务未在运行")
|
||||
return
|
||||
}
|
||||
|
||||
gs.manager.StopGoogleIndexScheduler()
|
||||
utils.Info("Google索引调度任务已停止")
|
||||
}
|
||||
|
||||
// IsGoogleIndexSchedulerRunning 检查Google索引调度任务是否在运行
|
||||
func (gs *GlobalScheduler) IsGoogleIndexSchedulerRunning() bool {
|
||||
gs.mutex.RLock()
|
||||
defer gs.mutex.RUnlock()
|
||||
|
||||
return gs.manager.IsGoogleIndexRunning()
|
||||
}
|
||||
|
||||
541
scheduler/google_index.go
Normal file
541
scheduler/google_index.go
Normal file
@@ -0,0 +1,541 @@
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/ctwj/urldb/db/entity"
|
||||
"github.com/ctwj/urldb/db/repo"
|
||||
"github.com/ctwj/urldb/pkg/google"
|
||||
"github.com/ctwj/urldb/utils"
|
||||
)
|
||||
|
||||
// GoogleIndexScheduler Google索引调度器
|
||||
type GoogleIndexScheduler struct {
|
||||
*BaseScheduler
|
||||
config entity.SystemConfig
|
||||
stopChan chan bool
|
||||
isRunning bool
|
||||
enabled bool
|
||||
checkInterval time.Duration
|
||||
googleClient *google.Client
|
||||
taskItemRepo repo.TaskItemRepository
|
||||
taskRepo repo.TaskRepository
|
||||
|
||||
// 批量处理相关
|
||||
pendingURLResults []*repo.URLStatusResult
|
||||
currentTaskID uint
|
||||
}
|
||||
|
||||
// NewGoogleIndexScheduler 创建Google索引调度器
|
||||
func NewGoogleIndexScheduler(baseScheduler *BaseScheduler, taskItemRepo repo.TaskItemRepository, taskRepo repo.TaskRepository) *GoogleIndexScheduler {
|
||||
return &GoogleIndexScheduler{
|
||||
BaseScheduler: baseScheduler,
|
||||
taskItemRepo: taskItemRepo,
|
||||
taskRepo: taskRepo,
|
||||
stopChan: make(chan bool),
|
||||
isRunning: false,
|
||||
pendingURLResults: make([]*repo.URLStatusResult, 0),
|
||||
}
|
||||
}
|
||||
|
||||
// Start 启动Google索引调度任务
|
||||
func (s *GoogleIndexScheduler) Start() {
|
||||
if s.isRunning {
|
||||
utils.Debug("Google索引调度任务已在运行中")
|
||||
return
|
||||
}
|
||||
|
||||
// 加载配置
|
||||
if err := s.loadConfig(); err != nil {
|
||||
utils.Error("加载Google索引配置失败: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if !s.enabled {
|
||||
utils.Debug("Google索引功能未启用,跳过调度任务")
|
||||
return
|
||||
}
|
||||
|
||||
s.isRunning = true
|
||||
utils.Info("开始启动Google索引调度任务,检查间隔: %v", s.checkInterval)
|
||||
|
||||
go s.run()
|
||||
}
|
||||
|
||||
// Stop 停止Google索引调度任务
|
||||
func (s *GoogleIndexScheduler) Stop() {
|
||||
if !s.isRunning {
|
||||
return
|
||||
}
|
||||
|
||||
utils.Info("正在停止Google索引调度任务...")
|
||||
s.stopChan <- true
|
||||
s.isRunning = false
|
||||
}
|
||||
|
||||
// IsRunning 检查调度器是否正在运行
|
||||
func (s *GoogleIndexScheduler) IsRunning() bool {
|
||||
return s.isRunning
|
||||
}
|
||||
|
||||
// run 运行调度器主循环
|
||||
func (s *GoogleIndexScheduler) run() {
|
||||
ticker := time.NewTicker(s.checkInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
// 启动时立即执行一次
|
||||
s.performScheduledTasks()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-s.stopChan:
|
||||
utils.Info("Google索引调度任务已停止")
|
||||
return
|
||||
case <-ticker.C:
|
||||
s.performScheduledTasks()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// loadConfig 加载配置
|
||||
func (s *GoogleIndexScheduler) loadConfig() error {
|
||||
// 获取启用状态
|
||||
enabledStr, err := s.systemConfigRepo.GetConfigValue(entity.GoogleIndexConfigKeyEnabled)
|
||||
if err != nil {
|
||||
s.enabled = false
|
||||
} else {
|
||||
s.enabled = enabledStr == "true" || enabledStr == "1"
|
||||
}
|
||||
|
||||
// 获取检查间隔
|
||||
intervalStr, err := s.systemConfigRepo.GetConfigValue(entity.GoogleIndexConfigKeyCheckInterval)
|
||||
if err != nil {
|
||||
s.checkInterval = 60 * time.Minute // 默认60分钟
|
||||
} else {
|
||||
if interval, parseErr := time.ParseDuration(intervalStr + "m"); parseErr == nil {
|
||||
s.checkInterval = interval
|
||||
} else {
|
||||
s.checkInterval = 60 * time.Minute
|
||||
}
|
||||
}
|
||||
|
||||
// 初始化Google客户端
|
||||
if s.enabled {
|
||||
if err := s.initGoogleClient(); err != nil {
|
||||
utils.Error("初始化Google客户端失败: %v", err)
|
||||
s.enabled = false
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// initGoogleClient 初始化Google客户端
|
||||
func (s *GoogleIndexScheduler) initGoogleClient() error {
|
||||
// 获取凭据文件路径
|
||||
credentialsFile, err := s.systemConfigRepo.GetConfigValue(entity.GoogleIndexConfigKeyCredentialsFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("获取凭据文件路径失败: %v", err)
|
||||
}
|
||||
|
||||
// 获取站点URL,使用通用站点URL配置
|
||||
siteURL, err := s.systemConfigRepo.GetConfigValue(entity.ConfigKeyWebsiteURL)
|
||||
if err != nil || siteURL == "" || siteURL == "https://example.com" {
|
||||
siteURL = "https://pan.l9.lc" // 默认站点URL
|
||||
}
|
||||
|
||||
// 创建Google客户端配置
|
||||
config := &google.Config{
|
||||
CredentialsFile: credentialsFile,
|
||||
SiteURL: siteURL,
|
||||
}
|
||||
|
||||
client, err := google.NewClient(config)
|
||||
if err != nil {
|
||||
return fmt.Errorf("创建Google客户端失败: %v", err)
|
||||
}
|
||||
|
||||
s.googleClient = client
|
||||
return nil
|
||||
}
|
||||
|
||||
// performScheduledTasks 执行调度任务
|
||||
func (s *GoogleIndexScheduler) performScheduledTasks() {
|
||||
if !s.enabled {
|
||||
return
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// 任务0: 清理旧记录
|
||||
if err := s.taskItemRepo.CleanupOldRecords(); err != nil {
|
||||
utils.Error("清理旧记录失败: %v", err)
|
||||
}
|
||||
|
||||
// 任务1: 定期提交sitemap给Google
|
||||
if err := s.submitSitemapToGoogle(ctx); err != nil {
|
||||
utils.Error("提交sitemap失败: %v", err)
|
||||
}
|
||||
|
||||
// 任务2: 刷新待处理的URL结果
|
||||
s.flushURLResults()
|
||||
|
||||
utils.Debug("Google索引调度任务执行完成")
|
||||
}
|
||||
|
||||
// submitSitemapToGoogle 提交sitemap给Google
|
||||
func (s *GoogleIndexScheduler) submitSitemapToGoogle(ctx context.Context) error {
|
||||
utils.Info("开始提交sitemap给Google...")
|
||||
|
||||
// 获取站点URL构建sitemap URL
|
||||
siteURL, err := s.systemConfigRepo.GetConfigValue(entity.ConfigKeyWebsiteURL)
|
||||
if err != nil || siteURL == "" || siteURL == "https://example.com" {
|
||||
siteURL = "https://pan.l9.lc" // 默认站点URL
|
||||
}
|
||||
|
||||
sitemapURL := siteURL
|
||||
if !strings.HasSuffix(sitemapURL, "/") {
|
||||
sitemapURL += "/"
|
||||
}
|
||||
sitemapURL += "sitemap.xml"
|
||||
|
||||
utils.Info("提交sitemap: %s", sitemapURL)
|
||||
|
||||
// 验证sitemapURL不为空
|
||||
if sitemapURL == "" || sitemapURL == "/sitemap.xml" {
|
||||
return fmt.Errorf("网站地图URL不能为空")
|
||||
}
|
||||
|
||||
// 提交sitemap给Google
|
||||
err = s.googleClient.SubmitSitemap(sitemapURL)
|
||||
if err != nil {
|
||||
utils.Error("提交sitemap失败: %s, 错误: %v", sitemapURL, err)
|
||||
return fmt.Errorf("提交sitemap失败: %v", err)
|
||||
}
|
||||
|
||||
utils.Info("sitemap提交成功: %s", sitemapURL)
|
||||
return nil
|
||||
}
|
||||
|
||||
// scanAndSubmitUnindexedURLs 扫描并提交未索引的URL
|
||||
func (s *GoogleIndexScheduler) scanAndSubmitUnindexedURLs(ctx context.Context) error {
|
||||
utils.Info("开始扫描未索引的URL...")
|
||||
|
||||
// 1. 获取所有资源URL
|
||||
resources, err := s.resourceRepo.GetAllValidResources()
|
||||
if err != nil {
|
||||
return fmt.Errorf("获取资源列表失败: %v", err)
|
||||
}
|
||||
|
||||
// 2. 获取已索引的URL记录
|
||||
indexedURLs, err := s.getIndexedURLs()
|
||||
if err != nil {
|
||||
return fmt.Errorf("获取已索引URL列表失败: %v", err)
|
||||
}
|
||||
|
||||
// 3. 找出未索引的URL
|
||||
var unindexedURLs []string
|
||||
indexedURLSet := make(map[string]bool)
|
||||
for _, url := range indexedURLs {
|
||||
indexedURLSet[url] = true
|
||||
}
|
||||
|
||||
for _, resource := range resources {
|
||||
if resource.IsPublic && resource.IsValid && resource.Key != "" {
|
||||
// 构建本站URL,而不是使用原始的外链URL
|
||||
siteURL, _ := s.systemConfigRepo.GetConfigValue(entity.ConfigKeyWebsiteURL)
|
||||
if siteURL == "" {
|
||||
siteURL = "https://pan.l9.lc" // 默认站点URL
|
||||
}
|
||||
localURL := fmt.Sprintf("%s/r/%s", siteURL, resource.Key)
|
||||
|
||||
if !indexedURLSet[localURL] {
|
||||
unindexedURLs = append(unindexedURLs, localURL)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
utils.Info("发现 %d 个未索引的URL", len(unindexedURLs))
|
||||
|
||||
// 4. 批量提交未索引的URL
|
||||
if len(unindexedURLs) > 0 {
|
||||
if err := s.batchSubmitURLs(ctx, unindexedURLs); err != nil {
|
||||
return fmt.Errorf("批量提交URL失败: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// getIndexedURLs 获取已索引的URL列表
|
||||
func (s *GoogleIndexScheduler) getIndexedURLs() ([]string, error) {
|
||||
return s.taskItemRepo.GetDistinctProcessedURLs()
|
||||
}
|
||||
|
||||
// batchSubmitURLs 批量提交URL
|
||||
func (s *GoogleIndexScheduler) batchSubmitURLs(ctx context.Context, urls []string) error {
|
||||
utils.Info("开始批量提交 %d 个URL到Google索引...", len(urls))
|
||||
|
||||
// 获取批量大小配置
|
||||
batchSizeStr, _ := s.systemConfigRepo.GetConfigValue(entity.GoogleIndexConfigKeyBatchSize)
|
||||
batchSize := 100 // 默认值
|
||||
if batchSizeStr != "" {
|
||||
if size, err := strconv.Atoi(batchSizeStr); err == nil && size > 0 {
|
||||
batchSize = size
|
||||
}
|
||||
}
|
||||
|
||||
// 获取并发数配置
|
||||
concurrencyStr, _ := s.systemConfigRepo.GetConfigValue(entity.GoogleIndexConfigKeyConcurrency)
|
||||
concurrency := 5 // 默认值
|
||||
if concurrencyStr != "" {
|
||||
if conc, err := strconv.Atoi(concurrencyStr); err == nil && conc > 0 {
|
||||
concurrency = conc
|
||||
}
|
||||
}
|
||||
|
||||
// 分批处理
|
||||
for i := 0; i < len(urls); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(urls) {
|
||||
end = len(urls)
|
||||
}
|
||||
|
||||
batch := urls[i:end]
|
||||
if err := s.processBatch(ctx, batch, concurrency); err != nil {
|
||||
utils.Error("处理批次失败 (批次 %d-%d): %v", i+1, end, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// 避免API限制,批次间稍作延迟
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
|
||||
utils.Info("批量URL提交完成")
|
||||
return nil
|
||||
}
|
||||
|
||||
// processBatch 处理单个批次
|
||||
func (s *GoogleIndexScheduler) processBatch(ctx context.Context, urls []string, concurrency int) error {
|
||||
semaphore := make(chan struct{}, concurrency)
|
||||
errChan := make(chan error, len(urls))
|
||||
|
||||
for _, url := range urls {
|
||||
go func(u string) {
|
||||
semaphore <- struct{}{}
|
||||
defer func() { <-semaphore }()
|
||||
|
||||
// 检查URL索引状态
|
||||
result, err := s.googleClient.InspectURL(u)
|
||||
if err != nil {
|
||||
utils.Error("检查URL失败: %s, 错误: %v", u, err)
|
||||
errChan <- err
|
||||
return
|
||||
}
|
||||
|
||||
// Google Search Console API 不直接支持URL提交
|
||||
// 这里只记录URL状态,实际的URL索引需要通过sitemap或其他方式
|
||||
if result.IndexStatusResult.IndexingState == "NOT_SUBMITTED" {
|
||||
utils.Debug("URL未提交,需要通过sitemap提交: %s", u)
|
||||
// TODO: 可以考虑将未提交的URL加入到sitemap中
|
||||
}
|
||||
|
||||
// 记录索引状态
|
||||
s.recordURLStatus(u, result)
|
||||
errChan <- nil
|
||||
}(url)
|
||||
}
|
||||
|
||||
// 等待所有goroutine完成
|
||||
for i := 0; i < len(urls); i++ {
|
||||
if err := <-errChan; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkIndexedURLsStatus 检查已索引URL的状态
|
||||
func (s *GoogleIndexScheduler) checkIndexedURLsStatus(ctx context.Context) error {
|
||||
utils.Info("开始检查已索引URL的状态...")
|
||||
|
||||
// 暂时跳过状态检查,因为需要TaskItemRepository访问权限
|
||||
// TODO: 后续通过扩展BaseScheduler来支持TaskItemRepository
|
||||
urlsToCheck := []string{}
|
||||
utils.Info("检查 %d 个已索引URL的状态", len(urlsToCheck))
|
||||
|
||||
// 并发检查状态
|
||||
concurrencyStr, _ := s.systemConfigRepo.GetConfigValue(entity.GoogleIndexConfigKeyConcurrency)
|
||||
concurrency := 3 // 状态检查使用较低并发
|
||||
if concurrencyStr != "" {
|
||||
if conc, err := strconv.Atoi(concurrencyStr); err == nil && conc > 0 {
|
||||
concurrency = conc / 2 // 状态检查并发减半
|
||||
if concurrency < 1 {
|
||||
concurrency = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 由于没有URL需要检查,跳过循环
|
||||
if len(urlsToCheck) == 0 {
|
||||
utils.Info("没有URL需要状态检查")
|
||||
return nil
|
||||
}
|
||||
|
||||
semaphore := make(chan struct{}, concurrency)
|
||||
for _, url := range urlsToCheck {
|
||||
go func(u string) {
|
||||
semaphore <- struct{}{}
|
||||
defer func() { <-semaphore }()
|
||||
|
||||
// 检查URL最新状态
|
||||
result, err := s.googleClient.InspectURL(u)
|
||||
if err != nil {
|
||||
utils.Error("检查URL状态失败: %s, 错误: %v", u, err)
|
||||
return
|
||||
}
|
||||
|
||||
// 记录状态
|
||||
s.recordURLStatus(u, result)
|
||||
}(url)
|
||||
}
|
||||
|
||||
// 等待所有检查完成
|
||||
for i := 0; i < len(urlsToCheck); i++ {
|
||||
<-semaphore
|
||||
}
|
||||
|
||||
utils.Info("索引状态检查完成")
|
||||
return nil
|
||||
}
|
||||
|
||||
// recordURLStatus 记录URL索引状态
|
||||
func (s *GoogleIndexScheduler) recordURLStatus(url string, result *google.URLInspectionResult) {
|
||||
// 构造结果对象
|
||||
urlResult := &repo.URLStatusResult{
|
||||
URL: url,
|
||||
IndexStatus: result.IndexStatusResult.IndexingState,
|
||||
InspectResult: s.formatInspectResult(result),
|
||||
MobileFriendly: s.getMobileFriendly(result),
|
||||
StatusCode: s.getStatusCode(result),
|
||||
LastCrawled: s.parseLastCrawled(result),
|
||||
ErrorMessage: s.getErrorMessage(result),
|
||||
}
|
||||
|
||||
// 暂存到批量处理列表,定期批量写入
|
||||
s.pendingURLResults = append(s.pendingURLResults, urlResult)
|
||||
|
||||
// 达到批量大小时写入数据库
|
||||
if len(s.pendingURLResults) >= 50 {
|
||||
s.flushURLResults()
|
||||
}
|
||||
}
|
||||
|
||||
// updateURLStatus 更新URL状态
|
||||
func (s *GoogleIndexScheduler) updateURLStatus(taskItem *entity.TaskItem, result *google.URLInspectionResult) {
|
||||
// 暂时只记录日志,不保存到数据库
|
||||
// TODO: 后续通过扩展BaseScheduler来支持TaskItemRepository以保存状态
|
||||
utils.Debug("更新URL状态: %s - %s", taskItem.URL, result.IndexStatusResult.IndexingState)
|
||||
}
|
||||
|
||||
// flushURLResults 批量写入URL结果
|
||||
func (s *GoogleIndexScheduler) flushURLResults() {
|
||||
if len(s.pendingURLResults) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// 如果没有当前任务,创建一个汇总任务
|
||||
if s.currentTaskID == 0 {
|
||||
task := &entity.Task{
|
||||
Title: fmt.Sprintf("自动索引检查 - %s", time.Now().Format("2006-01-02 15:04:05")),
|
||||
Type: entity.TaskTypeGoogleIndex,
|
||||
Status: entity.TaskStatusCompleted,
|
||||
Description: fmt.Sprintf("自动检查并更新 %d 个URL的索引状态", len(s.pendingURLResults)),
|
||||
TotalItems: len(s.pendingURLResults),
|
||||
Progress: 100.0,
|
||||
}
|
||||
|
||||
if err := s.taskRepo.Create(task); err != nil {
|
||||
utils.Error("创建汇总任务失败: %v", err)
|
||||
return
|
||||
}
|
||||
s.currentTaskID = task.ID
|
||||
}
|
||||
|
||||
// 批量写入URL状态
|
||||
if err := s.taskItemRepo.UpsertURLStatusRecords(s.currentTaskID, s.pendingURLResults); err != nil {
|
||||
utils.Error("批量写入URL状态失败: %v", err)
|
||||
} else {
|
||||
utils.Info("批量写入URL状态成功: %d 个", len(s.pendingURLResults))
|
||||
}
|
||||
|
||||
// 清空待处理列表
|
||||
s.pendingURLResults = s.pendingURLResults[:0]
|
||||
}
|
||||
|
||||
// 辅助方法:格式化检查结果
|
||||
func (s *GoogleIndexScheduler) formatInspectResult(result *google.URLInspectionResult) string {
|
||||
if result == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
data, err := json.Marshal(result)
|
||||
if err != nil {
|
||||
return fmt.Sprintf("Error: %v", err)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
// 辅助方法:获取移动友好状态
|
||||
func (s *GoogleIndexScheduler) getMobileFriendly(result *google.URLInspectionResult) bool {
|
||||
if result != nil {
|
||||
return result.MobileUsabilityResult.MobileFriendly
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// 辅助方法:获取状态码
|
||||
func (s *GoogleIndexScheduler) getStatusCode(result *google.URLInspectionResult) int {
|
||||
if result != nil {
|
||||
// 这里可以根据实际的Google API响应结构来获取状态码
|
||||
// 暂时返回200表示成功
|
||||
return 200
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// 辅助方法:解析最后抓取时间
|
||||
func (s *GoogleIndexScheduler) parseLastCrawled(result *google.URLInspectionResult) *time.Time {
|
||||
if result != nil && result.IndexStatusResult.LastCrawled != "" {
|
||||
// 这里需要根据实际的Google API响应结构来解析时间
|
||||
// 暂时返回当前时间
|
||||
now := time.Now()
|
||||
return &now
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// 辅助方法:获取错误信息
|
||||
func (s *GoogleIndexScheduler) getErrorMessage(result *google.URLInspectionResult) string {
|
||||
if result != nil {
|
||||
// 根据索引状态判断是否有错误
|
||||
if result.IndexStatusResult.IndexingState == "ERROR" {
|
||||
return "索引状态错误"
|
||||
}
|
||||
if result.IndexStatusResult.IndexingState == "NOT_FOUND" {
|
||||
return "页面未找到"
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// SetRunning 设置运行状态
|
||||
func (s *GoogleIndexScheduler) SetRunning(running bool) {
|
||||
s.isRunning = running
|
||||
}
|
||||
@@ -11,6 +11,7 @@ type Manager struct {
|
||||
hotDramaScheduler *HotDramaScheduler
|
||||
readyResourceScheduler *ReadyResourceScheduler
|
||||
sitemapScheduler *SitemapScheduler
|
||||
googleIndexScheduler *GoogleIndexScheduler
|
||||
}
|
||||
|
||||
// NewManager 创建调度器管理器
|
||||
@@ -23,6 +24,8 @@ func NewManager(
|
||||
cksRepo repo.CksRepository,
|
||||
tagRepo repo.TagRepository,
|
||||
categoryRepo repo.CategoryRepository,
|
||||
taskItemRepo repo.TaskItemRepository,
|
||||
taskRepo repo.TaskRepository,
|
||||
) *Manager {
|
||||
// 创建基础调度器
|
||||
baseScheduler := NewBaseScheduler(
|
||||
@@ -40,12 +43,14 @@ func NewManager(
|
||||
hotDramaScheduler := NewHotDramaScheduler(baseScheduler)
|
||||
readyResourceScheduler := NewReadyResourceScheduler(baseScheduler)
|
||||
sitemapScheduler := NewSitemapScheduler(baseScheduler)
|
||||
googleIndexScheduler := NewGoogleIndexScheduler(baseScheduler, taskItemRepo, taskRepo)
|
||||
|
||||
return &Manager{
|
||||
baseScheduler: baseScheduler,
|
||||
hotDramaScheduler: hotDramaScheduler,
|
||||
readyResourceScheduler: readyResourceScheduler,
|
||||
sitemapScheduler: sitemapScheduler,
|
||||
googleIndexScheduler: googleIndexScheduler,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,6 +64,9 @@ func (m *Manager) StartAll() {
|
||||
// 启动待处理资源调度任务
|
||||
m.readyResourceScheduler.Start()
|
||||
|
||||
// 启动Google索引调度任务
|
||||
m.googleIndexScheduler.Start()
|
||||
|
||||
utils.Debug("所有调度任务已启动")
|
||||
}
|
||||
|
||||
@@ -72,6 +80,9 @@ func (m *Manager) StopAll() {
|
||||
// 停止待处理资源调度任务
|
||||
m.readyResourceScheduler.Stop()
|
||||
|
||||
// 停止Google索引调度任务
|
||||
m.googleIndexScheduler.Stop()
|
||||
|
||||
utils.Debug("所有调度任务已停止")
|
||||
}
|
||||
|
||||
@@ -140,11 +151,27 @@ func (m *Manager) TriggerSitemapGeneration() {
|
||||
go m.sitemapScheduler.generateSitemap()
|
||||
}
|
||||
|
||||
// StartGoogleIndexScheduler 启动Google索引调度任务
|
||||
func (m *Manager) StartGoogleIndexScheduler() {
|
||||
m.googleIndexScheduler.Start()
|
||||
}
|
||||
|
||||
// StopGoogleIndexScheduler 停止Google索引调度任务
|
||||
func (m *Manager) StopGoogleIndexScheduler() {
|
||||
m.googleIndexScheduler.Stop()
|
||||
}
|
||||
|
||||
// IsGoogleIndexRunning 检查Google索引调度任务是否在运行
|
||||
func (m *Manager) IsGoogleIndexRunning() bool {
|
||||
return m.googleIndexScheduler.IsRunning()
|
||||
}
|
||||
|
||||
// GetStatus 获取所有调度任务的状态
|
||||
func (m *Manager) GetStatus() map[string]bool {
|
||||
return map[string]bool{
|
||||
"hot_drama": m.IsHotDramaRunning(),
|
||||
"ready_resource": m.IsReadyResourceRunning(),
|
||||
"sitemap": m.IsSitemapRunning(),
|
||||
"google_index": m.IsGoogleIndexRunning(),
|
||||
}
|
||||
}
|
||||
|
||||
404
task/google_index_processor.go
Normal file
404
task/google_index_processor.go
Normal file
@@ -0,0 +1,404 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/ctwj/urldb/db/entity"
|
||||
"github.com/ctwj/urldb/db/repo"
|
||||
"github.com/ctwj/urldb/pkg/google"
|
||||
"github.com/ctwj/urldb/utils"
|
||||
)
|
||||
|
||||
// GoogleIndexProcessor Google索引任务处理器
|
||||
type GoogleIndexProcessor struct {
|
||||
repoMgr *repo.RepositoryManager
|
||||
client *google.Client
|
||||
config *GoogleIndexProcessorConfig
|
||||
}
|
||||
|
||||
// GoogleIndexProcessorConfig Google索引处理器配置
|
||||
type GoogleIndexProcessorConfig struct {
|
||||
CredentialsFile string
|
||||
SiteURL string
|
||||
TokenFile string
|
||||
Concurrency int
|
||||
RetryAttempts int
|
||||
RetryDelay time.Duration
|
||||
}
|
||||
|
||||
// GoogleIndexTaskInput Google索引任务输入数据结构
|
||||
type GoogleIndexTaskInput struct {
|
||||
URLs []string `json:"urls"`
|
||||
Operation string `json:"operation"` // indexing_check, sitemap_submit, batch_index
|
||||
SitemapURL string `json:"sitemap_url,omitempty"`
|
||||
}
|
||||
|
||||
// GoogleIndexTaskOutput Google索引任务输出数据结构
|
||||
type GoogleIndexTaskOutput struct {
|
||||
URL string `json:"url,omitempty"`
|
||||
IndexStatus string `json:"index_status,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Success bool `json:"success"`
|
||||
Message string `json:"message"`
|
||||
Time string `json:"time"`
|
||||
Result *google.URLInspectionResult `json:"result,omitempty"`
|
||||
}
|
||||
|
||||
// NewGoogleIndexProcessor 创建Google索引任务处理器
|
||||
func NewGoogleIndexProcessor(repoMgr *repo.RepositoryManager) *GoogleIndexProcessor {
|
||||
return &GoogleIndexProcessor{
|
||||
repoMgr: repoMgr,
|
||||
config: &GoogleIndexProcessorConfig{
|
||||
RetryAttempts: 3,
|
||||
RetryDelay: 2 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// GetTaskType 获取任务类型
|
||||
func (gip *GoogleIndexProcessor) GetTaskType() string {
|
||||
return "google_index"
|
||||
}
|
||||
|
||||
// Process 处理Google索引任务项
|
||||
func (gip *GoogleIndexProcessor) Process(ctx context.Context, taskID uint, item *entity.TaskItem) error {
|
||||
utils.Info("开始处理Google索引任务项: %d", item.ID)
|
||||
|
||||
// 解析输入数据
|
||||
var input GoogleIndexTaskInput
|
||||
if err := json.Unmarshal([]byte(item.InputData), &input); err != nil {
|
||||
utils.Error("解析输入数据失败: %v", err)
|
||||
gip.updateTaskItemStatus(item, entity.TaskItemStatusFailed, "", false, nil, 400, err.Error())
|
||||
return fmt.Errorf("解析输入数据失败: %v", err)
|
||||
}
|
||||
|
||||
// 初始化Google客户端
|
||||
client, err := gip.initGoogleClient()
|
||||
if err != nil {
|
||||
utils.Error("初始化Google客户端失败: %v", err)
|
||||
gip.updateTaskItemStatus(item, entity.TaskItemStatusFailed, "", false, nil, 500, err.Error())
|
||||
return fmt.Errorf("初始化Google客户端失败: %v", err)
|
||||
}
|
||||
|
||||
// 根据操作类型执行不同任务
|
||||
switch input.Operation {
|
||||
case "url_indexing":
|
||||
return gip.processURLIndexing(ctx, client, taskID, item, input)
|
||||
case "sitemap_submit":
|
||||
return gip.processSitemapSubmit(ctx, client, taskID, item, input)
|
||||
case "status_check":
|
||||
return gip.processStatusCheck(ctx, client, taskID, item, input)
|
||||
default:
|
||||
errorMsg := fmt.Sprintf("不支持的操作类型: %s", input.Operation)
|
||||
gip.updateTaskItemStatus(item, entity.TaskItemStatusFailed, "", false, nil, 400, errorMsg)
|
||||
return fmt.Errorf(errorMsg)
|
||||
}
|
||||
}
|
||||
|
||||
// processURLIndexing 处理URL索引检查
|
||||
func (gip *GoogleIndexProcessor) processURLIndexing(ctx context.Context, client *google.Client, taskID uint, item *entity.TaskItem, input GoogleIndexTaskInput) error {
|
||||
utils.Info("开始URL索引检查: %v", input.URLs)
|
||||
|
||||
for _, url := range input.URLs {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
gip.updateTaskItemStatus(item, entity.TaskItemStatusFailed, "", false, nil, 0, "任务被取消")
|
||||
return ctx.Err()
|
||||
default:
|
||||
// 检查URL索引状态
|
||||
result, err := gip.inspectURL(client, url)
|
||||
if err != nil {
|
||||
utils.Error("检查URL索引状态失败: %s, 错误: %v", url, err)
|
||||
gip.updateTaskItemStatus(item, entity.TaskItemStatusFailed, "", false, nil, 500, err.Error())
|
||||
continue
|
||||
}
|
||||
|
||||
// 更新任务项状态
|
||||
var lastCrawled *time.Time
|
||||
if result.IndexStatusResult.LastCrawled != "" {
|
||||
parsedTime, err := time.Parse(time.RFC3339, result.IndexStatusResult.LastCrawled)
|
||||
if err == nil {
|
||||
lastCrawled = &parsedTime
|
||||
}
|
||||
}
|
||||
|
||||
gip.updateTaskItemStatus(item, entity.TaskItemStatusSuccess, result.IndexStatusResult.IndexingState, result.MobileUsabilityResult.MobileFriendly, lastCrawled, 200, "")
|
||||
|
||||
// 更新URL状态记录
|
||||
gip.updateURLStatus(url, result.IndexStatusResult.IndexingState, lastCrawled)
|
||||
|
||||
// 添加延迟避免API限制
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
utils.Info("URL索引检查完成")
|
||||
return nil
|
||||
}
|
||||
|
||||
// processSitemapSubmit 处理网站地图提交
|
||||
func (gip *GoogleIndexProcessor) processSitemapSubmit(ctx context.Context, client *google.Client, taskID uint, item *entity.TaskItem, input GoogleIndexTaskInput) error {
|
||||
utils.Info("开始网站地图提交: %s", input.SitemapURL)
|
||||
|
||||
if input.SitemapURL == "" {
|
||||
errorMsg := "网站地图URL不能为空"
|
||||
gip.updateTaskItemStatus(item, entity.TaskItemStatusFailed, "", false, nil, 400, errorMsg)
|
||||
return fmt.Errorf(errorMsg)
|
||||
}
|
||||
|
||||
// 提交网站地图
|
||||
err := client.SubmitSitemap(input.SitemapURL)
|
||||
if err != nil {
|
||||
utils.Error("提交网站地图失败: %s, 错误: %v", input.SitemapURL, err)
|
||||
gip.updateTaskItemStatus(item, entity.TaskItemStatusFailed, "", false, nil, 500, err.Error())
|
||||
return fmt.Errorf("提交网站地图失败: %v", err)
|
||||
}
|
||||
|
||||
// 更新任务项状态
|
||||
now := time.Now()
|
||||
gip.updateTaskItemStatus(item, entity.TaskItemStatusSuccess, "SUBMITTED", false, &now, 200, "")
|
||||
|
||||
utils.Info("网站地图提交完成: %s", input.SitemapURL)
|
||||
return nil
|
||||
}
|
||||
|
||||
// processStatusCheck 处理状态检查
|
||||
func (gip *GoogleIndexProcessor) processStatusCheck(ctx context.Context, client *google.Client, taskID uint, item *entity.TaskItem, input GoogleIndexTaskInput) error {
|
||||
utils.Info("开始状态检查: %v", input.URLs)
|
||||
|
||||
for _, url := range input.URLs {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
gip.updateTaskItemStatus(item, entity.TaskItemStatusFailed, "", false, nil, 0, "任务被取消")
|
||||
return ctx.Err()
|
||||
default:
|
||||
// 检查URL状态
|
||||
result, err := gip.inspectURL(client, url)
|
||||
if err != nil {
|
||||
utils.Error("检查URL状态失败: %s, 错误: %v", url, err)
|
||||
gip.updateTaskItemStatus(item, entity.TaskItemStatusFailed, "", false, nil, 500, err.Error())
|
||||
continue
|
||||
}
|
||||
|
||||
// 更新任务项状态
|
||||
var lastCrawled *time.Time
|
||||
if result.IndexStatusResult.LastCrawled != "" {
|
||||
parsedTime, err := time.Parse(time.RFC3339, result.IndexStatusResult.LastCrawled)
|
||||
if err == nil {
|
||||
lastCrawled = &parsedTime
|
||||
}
|
||||
}
|
||||
|
||||
gip.updateTaskItemStatus(item, entity.TaskItemStatusSuccess, result.IndexStatusResult.IndexingState, result.MobileUsabilityResult.MobileFriendly, lastCrawled, 200, "")
|
||||
|
||||
utils.Info("URL状态检查完成: %s, 状态: %s", url, result.IndexStatusResult.IndexingState)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// initGoogleClient 初始化Google客户端
|
||||
func (gip *GoogleIndexProcessor) initGoogleClient() (*google.Client, error) {
|
||||
// 使用固定的凭据文件路径,与验证逻辑保持一致
|
||||
credentialsFile := "data/google_credentials.json"
|
||||
|
||||
// 检查凭据文件是否存在
|
||||
if _, err := os.Stat(credentialsFile); os.IsNotExist(err) {
|
||||
return nil, fmt.Errorf("Google凭据文件不存在: %s", credentialsFile)
|
||||
}
|
||||
|
||||
// 从配置中获取网站URL
|
||||
siteURL, err := gip.repoMgr.SystemConfigRepository.GetConfigValue(entity.ConfigKeyWebsiteURL)
|
||||
if err != nil || siteURL == "" || siteURL == "https://example.com" {
|
||||
return nil, fmt.Errorf("未配置网站URL或在站点配置中设置了默认值")
|
||||
}
|
||||
|
||||
config := &google.Config{
|
||||
CredentialsFile: credentialsFile,
|
||||
SiteURL: siteURL,
|
||||
TokenFile: "data/google_token.json", // 使用固定token文件名,放在data目录下
|
||||
}
|
||||
|
||||
client, err := google.NewClient(config)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("创建Google客户端失败: %v", err)
|
||||
}
|
||||
|
||||
return client, nil
|
||||
}
|
||||
|
||||
// inspectURL 检查URL索引状态
|
||||
func (gip *GoogleIndexProcessor) inspectURL(client *google.Client, url string) (*google.URLInspectionResult, error) {
|
||||
// 重试机制
|
||||
var result *google.URLInspectionResult
|
||||
var err error
|
||||
|
||||
for attempt := 0; attempt <= gip.config.RetryAttempts; attempt++ {
|
||||
result, err = client.InspectURL(url)
|
||||
if err == nil {
|
||||
break // 成功则退出重试循环
|
||||
}
|
||||
|
||||
if attempt < gip.config.RetryAttempts {
|
||||
utils.Info("URL检查失败,第%d次重试: %s, 错误: %v", attempt+1, url, err)
|
||||
time.Sleep(gip.config.RetryDelay)
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("检查URL失败: %v", err)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// updateTaskItemStatus 更新任务项状态
|
||||
func (gip *GoogleIndexProcessor) updateTaskItemStatus(item *entity.TaskItem, status entity.TaskItemStatus, indexStatus string, mobileFriendly bool, lastCrawled *time.Time, statusCode int, errorMessage string) {
|
||||
item.Status = status
|
||||
item.ErrorMessage = errorMessage
|
||||
|
||||
// 更新Google索引特有字段
|
||||
item.IndexStatus = indexStatus
|
||||
item.MobileFriendly = mobileFriendly
|
||||
item.LastCrawled = lastCrawled
|
||||
item.StatusCode = statusCode
|
||||
|
||||
now := time.Now()
|
||||
item.ProcessedAt = &now
|
||||
|
||||
// 保存更新
|
||||
if err := gip.repoMgr.TaskItemRepository.Update(item); err != nil {
|
||||
utils.Error("更新任务项状态失败: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// updateURLStatus 更新URL状态记录(使用任务项存储)
|
||||
func (gip *GoogleIndexProcessor) updateURLStatus(url string, indexStatus string, lastCrawled *time.Time) {
|
||||
// 在任务项中记录URL状态,而不是使用专门的URL状态表
|
||||
// 此功能现在通过任务系统中的TaskItem记录来跟踪
|
||||
utils.Debug("URL状态已更新: %s, 状态: %s", url, indexStatus)
|
||||
}
|
||||
|
||||
// BatchProcessURLs 批量处理URLs
|
||||
func (gip *GoogleIndexProcessor) BatchProcessURLs(ctx context.Context, urls []string, operation string, taskID uint) error {
|
||||
utils.Info("开始批量处理URLs,数量: %d, 操作: %s", len(urls), operation)
|
||||
|
||||
// 根据并发数创建工作池
|
||||
semaphore := make(chan struct{}, gip.config.Concurrency)
|
||||
errChan := make(chan error, len(urls))
|
||||
|
||||
for _, url := range urls {
|
||||
go func(u string) {
|
||||
semaphore <- struct{}{} // 获取信号量
|
||||
defer func() { <-semaphore }() // 释放信号量
|
||||
|
||||
// 处理单个URL
|
||||
client, err := gip.initGoogleClient()
|
||||
if err != nil {
|
||||
errChan <- fmt.Errorf("初始化客户端失败: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
result, err := gip.inspectURL(client, u)
|
||||
if err != nil {
|
||||
utils.Error("处理URL失败: %s, 错误: %v", u, err)
|
||||
errChan <- err
|
||||
return
|
||||
}
|
||||
|
||||
// 更新状态
|
||||
var lastCrawled *time.Time
|
||||
if result.IndexStatusResult.LastCrawled != "" {
|
||||
parsedTime, err := time.Parse(time.RFC3339, result.IndexStatusResult.LastCrawled)
|
||||
if err == nil {
|
||||
lastCrawled = &parsedTime
|
||||
}
|
||||
}
|
||||
|
||||
// 创建任务项记录
|
||||
now := time.Now()
|
||||
inputData := map[string]interface{}{
|
||||
"urls": []string{u},
|
||||
"operation": "url_indexing",
|
||||
}
|
||||
inputDataJSON, _ := json.Marshal(inputData)
|
||||
|
||||
taskItem := &entity.TaskItem{
|
||||
TaskID: taskID,
|
||||
Status: entity.TaskItemStatusSuccess,
|
||||
InputData: string(inputDataJSON),
|
||||
URL: u,
|
||||
IndexStatus: result.IndexStatusResult.IndexingState,
|
||||
MobileFriendly: result.MobileUsabilityResult.MobileFriendly,
|
||||
LastCrawled: lastCrawled,
|
||||
StatusCode: 200,
|
||||
ProcessedAt: &now,
|
||||
}
|
||||
|
||||
if err := gip.repoMgr.TaskItemRepository.Create(taskItem); err != nil {
|
||||
utils.Error("创建任务项失败: %v", err)
|
||||
}
|
||||
|
||||
// 更新URL状态
|
||||
gip.updateURLStatus(u, result.IndexStatusResult.IndexingState, lastCrawled)
|
||||
|
||||
errChan <- nil
|
||||
}(url)
|
||||
}
|
||||
|
||||
// 等待所有goroutine完成
|
||||
for i := 0; i < len(urls); i++ {
|
||||
err := <-errChan
|
||||
if err != nil {
|
||||
utils.Error("批量处理URL时出错: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
utils.Info("批量处理URLs完成")
|
||||
return nil
|
||||
}
|
||||
|
||||
// SubmitSitemap 提交网站地图
|
||||
func (gip *GoogleIndexProcessor) SubmitSitemap(ctx context.Context, sitemapURL string, taskID uint) error {
|
||||
utils.Info("开始提交网站地图: %s", sitemapURL)
|
||||
|
||||
client, err := gip.initGoogleClient()
|
||||
if err != nil {
|
||||
return fmt.Errorf("初始化Google客户端失败: %v", err)
|
||||
}
|
||||
|
||||
err = client.SubmitSitemap(sitemapURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("提交网站地图失败: %v", err)
|
||||
}
|
||||
|
||||
// 创建任务项记录
|
||||
now := time.Now()
|
||||
inputData := map[string]interface{}{
|
||||
"sitemap_url": sitemapURL,
|
||||
"operation": "sitemap_submit",
|
||||
}
|
||||
inputDataJSON, _ := json.Marshal(inputData)
|
||||
|
||||
taskItem := &entity.TaskItem{
|
||||
TaskID: taskID,
|
||||
Status: entity.TaskItemStatusSuccess,
|
||||
InputData: string(inputDataJSON),
|
||||
URL: sitemapURL,
|
||||
IndexStatus: "SUBMITTED",
|
||||
StatusCode: 200,
|
||||
ProcessedAt: &now,
|
||||
}
|
||||
|
||||
if err := gip.repoMgr.TaskItemRepository.Create(taskItem); err != nil {
|
||||
utils.Error("创建任务项失败: %v", err)
|
||||
}
|
||||
|
||||
utils.Info("网站地图提交完成: %s", sitemapURL)
|
||||
return nil
|
||||
}
|
||||
@@ -555,3 +555,71 @@ func (tm *TaskManager) RecoverRunningTasks() error {
|
||||
utils.Info("任务恢复完成,共恢复 %d 个任务", recoveredCount)
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreateTask 创建任务
|
||||
func (tm *TaskManager) CreateTask(taskType, name, description string, configID *uint) (*entity.Task, error) {
|
||||
// 验证任务类型是否有对应的处理器
|
||||
tm.mu.RLock()
|
||||
_, exists := tm.processors[taskType]
|
||||
tm.mu.RUnlock()
|
||||
|
||||
if !exists {
|
||||
return nil, fmt.Errorf("未找到任务类型 %s 的处理器", taskType)
|
||||
}
|
||||
|
||||
task := &entity.Task{
|
||||
Type: entity.TaskType(taskType),
|
||||
Name: name,
|
||||
Title: name, // 设置Title为相同值,保持兼容性
|
||||
Description: description,
|
||||
Status: entity.TaskStatusPending,
|
||||
ConfigID: configID,
|
||||
}
|
||||
|
||||
err := tm.repoMgr.TaskRepository.Create(task)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("创建任务失败: %v", err)
|
||||
}
|
||||
|
||||
utils.Info("创建任务成功: ID=%d, 类型=%s, 名称=%s", task.ID, task.Type, task.Name)
|
||||
return task, nil
|
||||
}
|
||||
|
||||
// GetTask 获取任务详情
|
||||
func (tm *TaskManager) GetTask(taskID uint) (*entity.Task, error) {
|
||||
task, err := tm.repoMgr.TaskRepository.GetByID(taskID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("获取任务失败: %v", err)
|
||||
}
|
||||
return task, nil
|
||||
}
|
||||
|
||||
// GetTaskItemStats 获取任务项统计信息
|
||||
func (tm *TaskManager) GetTaskItemStats(taskID uint) (map[string]int, error) {
|
||||
stats, err := tm.repoMgr.TaskItemRepository.GetStatsByTaskID(taskID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("获取任务项统计失败: %v", err)
|
||||
}
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
// QueryTaskItems 查询任务项
|
||||
func (tm *TaskManager) QueryTaskItems(taskID uint, page, pageSize int, status string) ([]*entity.TaskItem, int64, error) {
|
||||
items, total, err := tm.repoMgr.TaskItemRepository.GetListByTaskID(taskID, page, pageSize, status)
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("查询任务项失败: %v", err)
|
||||
}
|
||||
return items, total, nil
|
||||
}
|
||||
|
||||
// CreateTaskItems 创建任务项
|
||||
func (tm *TaskManager) CreateTaskItems(taskID uint, items []*entity.TaskItem) error {
|
||||
for _, item := range items {
|
||||
item.TaskID = taskID
|
||||
if err := tm.repoMgr.TaskItemRepository.Create(item); err != nil {
|
||||
return fmt.Errorf("创建任务项失败: %v", err)
|
||||
}
|
||||
}
|
||||
utils.Info("创建任务项成功: 任务ID=%d, 数量=%d", taskID, len(items))
|
||||
return nil
|
||||
}
|
||||
|
||||
552
web/components/Admin/GoogleIndexTab.vue
Normal file
552
web/components/Admin/GoogleIndexTab.vue
Normal file
@@ -0,0 +1,552 @@
|
||||
<template>
|
||||
<div class="tab-content-container">
|
||||
<div class="bg-white dark:bg-gray-800 rounded-lg shadow p-6">
|
||||
<!-- Google索引配置 -->
|
||||
<div class="mb-6">
|
||||
<h3 class="text-lg font-semibold text-gray-900 dark:text-white mb-2">Google索引配置</h3>
|
||||
<p class="text-gray-600 dark:text-gray-400">配置Google Search Console API和索引相关设置</p>
|
||||
</div>
|
||||
|
||||
<div class="mb-6 p-4 bg-gray-50 dark:bg-gray-700/50 rounded-lg">
|
||||
<div class="flex items-center justify-between mb-4">
|
||||
<div>
|
||||
<h4 class="font-medium text-gray-900 dark:text-white mb-2">Google索引功能</h4>
|
||||
<p class="text-sm text-gray-600 dark:text-gray-400">
|
||||
开启后系统将自动检查和提交URL到Google索引
|
||||
</p>
|
||||
</div>
|
||||
<n-switch
|
||||
v-model:value="googleIndexConfig.enabled"
|
||||
@update:value="updateGoogleIndexConfig"
|
||||
:loading="configLoading"
|
||||
size="large"
|
||||
>
|
||||
<template #checked>已开启</template>
|
||||
<template #unchecked>已关闭</template>
|
||||
</n-switch>
|
||||
</div>
|
||||
|
||||
<!-- 配置详情 -->
|
||||
<div class="border-t border-gray-200 dark:border-gray-600 pt-4">
|
||||
<div class="grid grid-cols-1 md:grid-cols-2 gap-4">
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">站点URL</label>
|
||||
<n-input
|
||||
:value="getSiteUrlDisplay()"
|
||||
:disabled="true"
|
||||
placeholder="请先在站点配置中设置站点URL"
|
||||
>
|
||||
<template #prefix>
|
||||
<i class="fas fa-globe text-gray-400"></i>
|
||||
</template>
|
||||
</n-input>
|
||||
<!-- 所有权验证按钮 -->
|
||||
<div class="mt-3">
|
||||
<n-button
|
||||
type="info"
|
||||
size="small"
|
||||
ghost
|
||||
@click="$emit('show-verification')"
|
||||
>
|
||||
<template #icon>
|
||||
<i class="fas fa-shield-alt"></i>
|
||||
</template>
|
||||
所有权验证
|
||||
</n-button>
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">凭据文件路径</label>
|
||||
<div class="flex flex-col space-y-2">
|
||||
<n-input
|
||||
:value="credentialsFilePath"
|
||||
placeholder="点击上传按钮选择文件"
|
||||
:disabled="true"
|
||||
/>
|
||||
<div class="flex space-x-2">
|
||||
<!-- 申请凭据按钮 -->
|
||||
<n-button
|
||||
size="small"
|
||||
type="warning"
|
||||
ghost
|
||||
@click="$emit('show-credentials-guide')"
|
||||
>
|
||||
<template #icon>
|
||||
<i class="fas fa-question-circle"></i>
|
||||
</template>
|
||||
申请凭据
|
||||
</n-button>
|
||||
<!-- 上传按钮 -->
|
||||
<n-button
|
||||
size="small"
|
||||
type="primary"
|
||||
ghost
|
||||
@click="$emit('select-credentials-file')"
|
||||
>
|
||||
<template #icon>
|
||||
<i class="fas fa-upload"></i>
|
||||
</template>
|
||||
上传凭据
|
||||
</n-button>
|
||||
<!-- 验证按钮 -->
|
||||
<n-button
|
||||
size="small"
|
||||
type="info"
|
||||
ghost
|
||||
@click="validateCredentials"
|
||||
:loading="validatingCredentials"
|
||||
:disabled="!credentialsFilePath"
|
||||
>
|
||||
<template #icon>
|
||||
<i class="fas fa-check-circle"></i>
|
||||
</template>
|
||||
验证凭据
|
||||
</n-button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">检查间隔(分钟)</label>
|
||||
<n-input-number
|
||||
v-model:value="googleIndexConfig.checkInterval"
|
||||
:min="1"
|
||||
:max="1440"
|
||||
@update:value="updateGoogleIndexConfig"
|
||||
:disabled="!credentialsFilePath"
|
||||
style="width: 100%"
|
||||
/>
|
||||
</div>
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">批处理大小</label>
|
||||
<n-input-number
|
||||
v-model:value="googleIndexConfig.batchSize"
|
||||
:min="1"
|
||||
:max="1000"
|
||||
@update:value="updateGoogleIndexConfig"
|
||||
:disabled="!credentialsFilePath"
|
||||
style="width: 100%"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 凭据状态 -->
|
||||
<div v-if="credentialsStatus" class="mt-4 p-3 rounded-lg border"
|
||||
:class="{
|
||||
'bg-green-50 border-green-200 text-green-700 dark:bg-green-900/20 dark:border-green-800 dark:text-green-300': credentialsStatus === 'valid',
|
||||
'bg-yellow-50 border-yellow-200 text-yellow-700 dark:bg-yellow-900/20 dark:border-yellow-800 dark:text-yellow-300': credentialsStatus === 'invalid',
|
||||
'bg-blue-50 border-blue-200 text-blue-700 dark:bg-blue-900/20 dark:border-blue-800 dark:text-blue-300': credentialsStatus === 'verifying'
|
||||
}"
|
||||
>
|
||||
<div class="flex items-center">
|
||||
<i
|
||||
:class="{
|
||||
'fas fa-check-circle text-green-500 dark:text-green-400': credentialsStatus === 'valid',
|
||||
'fas fa-exclamation-circle text-yellow-500 dark:text-yellow-400': credentialsStatus === 'invalid',
|
||||
'fas fa-spinner fa-spin text-blue-500 dark:text-blue-400': credentialsStatus === 'verifying'
|
||||
}"
|
||||
class="mr-2"
|
||||
></i>
|
||||
<span>{{ credentialsStatusMessage }}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Google索引统计 -->
|
||||
<div class="grid grid-cols-1 md:grid-cols-4 gap-4 mb-6">
|
||||
<div class="bg-blue-50 dark:bg-blue-900/20 rounded-lg p-4">
|
||||
<div class="flex items-center">
|
||||
<div class="p-2 bg-blue-100 dark:bg-blue-900 rounded-lg">
|
||||
<i class="fas fa-link text-blue-600 dark:text-blue-400"></i>
|
||||
</div>
|
||||
<div class="ml-3">
|
||||
<p class="text-sm text-gray-600 dark:text-gray-400">总URL数</p>
|
||||
<p class="text-xl font-bold text-gray-900 dark:text-white">{{ googleIndexStats.totalURLs || 0 }}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="bg-green-50 dark:bg-green-900/20 rounded-lg p-4">
|
||||
<div class="flex items-center">
|
||||
<div class="p-2 bg-green-100 dark:bg-green-900 rounded-lg">
|
||||
<i class="fas fa-check-circle text-green-600 dark:text-green-400"></i>
|
||||
</div>
|
||||
<div class="ml-3">
|
||||
<p class="text-sm text-gray-600 dark:text-gray-400">已索引</p>
|
||||
<p class="text-xl font-bold text-gray-900 dark:text-white">{{ googleIndexStats.indexedURLs || 0 }}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="bg-yellow-50 dark:bg-yellow-900/20 rounded-lg p-4">
|
||||
<div class="flex items-center">
|
||||
<div class="p-2 bg-yellow-100 dark:bg-yellow-900 rounded-lg">
|
||||
<i class="fas fa-exclamation-triangle text-yellow-600 dark:text-yellow-400"></i>
|
||||
</div>
|
||||
<div class="ml-3">
|
||||
<p class="text-sm text-gray-600 dark:text-gray-400">错误数</p>
|
||||
<p class="text-xl font-bold text-gray-900 dark:text-white">{{ googleIndexStats.errorURLs || 0 }}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="bg-purple-50 dark:bg-purple-900/20 rounded-lg p-4">
|
||||
<div class="flex items-center">
|
||||
<div class="p-2 bg-purple-100 dark:bg-purple-900 rounded-lg">
|
||||
<i class="fas fa-tasks text-purple-600 dark:text-purple-400"></i>
|
||||
</div>
|
||||
<div class="ml-3">
|
||||
<p class="text-sm text-gray-600 dark:text-gray-400">总任务数</p>
|
||||
<p class="text-xl font-bold text-gray-900 dark:text-white">{{ googleIndexStats.totalTasks || 0 }}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 外部工具链接 -->
|
||||
<div class="grid grid-cols-1 md:grid-cols-2 gap-4 mb-6">
|
||||
<div class="bg-purple-50 dark:bg-purple-900/20 rounded-lg p-4">
|
||||
<div class="flex items-center justify-between">
|
||||
<div class="flex items-center">
|
||||
<div class="p-2 bg-purple-100 dark:bg-purple-900 rounded-lg">
|
||||
<i class="fas fa-chart-line text-purple-600 dark:text-purple-400"></i>
|
||||
</div>
|
||||
<div class="ml-3">
|
||||
<p class="text-sm text-gray-600 dark:text-gray-400">Google Search Console</p>
|
||||
<p class="text-sm font-medium text-gray-900 dark:text-white">查看详细分析数据</p>
|
||||
</div>
|
||||
</div>
|
||||
<a
|
||||
:href="getSearchConsoleUrl()"
|
||||
target="_blank"
|
||||
class="px-3 py-1 bg-purple-600 text-white rounded-md hover:bg-purple-700 transition-colors text-sm"
|
||||
>
|
||||
打开控制台
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="bg-orange-50 dark:bg-orange-900/20 rounded-lg p-4">
|
||||
<div class="flex items-center justify-between">
|
||||
<div class="flex items-center">
|
||||
<div class="p-2 bg-orange-100 dark:bg-orange-900 rounded-lg">
|
||||
<i class="fas fa-chart-line text-orange-600 dark:text-orange-400"></i>
|
||||
</div>
|
||||
<div class="ml-3">
|
||||
<p class="text-sm text-gray-600 dark:text-gray-400">Google Analytics</p>
|
||||
<p class="text-sm font-medium text-gray-900 dark:text-white">网站流量分析仪表板</p>
|
||||
</div>
|
||||
</div>
|
||||
<a
|
||||
:href="getAnalyticsUrl()"
|
||||
target="_blank"
|
||||
class="px-3 py-1 bg-orange-600 text-white rounded-md hover:bg-orange-700 transition-colors text-sm"
|
||||
>
|
||||
查看分析
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 操作按钮 -->
|
||||
<div class="flex flex-wrap gap-3 mb-6">
|
||||
<n-button
|
||||
type="primary"
|
||||
@click="$emit('manual-check-urls')"
|
||||
:loading="manualCheckLoading"
|
||||
size="large"
|
||||
>
|
||||
<template #icon>
|
||||
<i class="fas fa-search"></i>
|
||||
</template>
|
||||
手动检查URL
|
||||
</n-button>
|
||||
|
||||
<n-button
|
||||
type="success"
|
||||
@click="submitSitemap"
|
||||
:loading="submitSitemapLoading"
|
||||
size="large"
|
||||
>
|
||||
<template #icon>
|
||||
<i class="fas fa-upload"></i>
|
||||
</template>
|
||||
提交网站地图
|
||||
</n-button>
|
||||
|
||||
<n-button
|
||||
type="info"
|
||||
@click="$emit('refresh-status')"
|
||||
size="large"
|
||||
>
|
||||
<template #icon>
|
||||
<i class="fas fa-sync-alt"></i>
|
||||
</template>
|
||||
刷新状态
|
||||
</n-button>
|
||||
</div>
|
||||
|
||||
<!-- 任务列表 -->
|
||||
<div>
|
||||
<h4 class="text-lg font-medium text-gray-900 dark:text-white mb-4">索引任务列表</h4>
|
||||
<n-data-table
|
||||
:columns="taskColumns"
|
||||
:data="tasks"
|
||||
:pagination="pagination"
|
||||
:loading="tasksLoading"
|
||||
:bordered="false"
|
||||
striped
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { useMessage } from 'naive-ui'
|
||||
import { useApi } from '~/composables/useApi'
|
||||
import { ref, computed, h, watch } from 'vue'
|
||||
|
||||
// Props
|
||||
interface Props {
|
||||
systemConfig?: any
|
||||
googleIndexConfig: any
|
||||
googleIndexStats: any
|
||||
tasks: any[]
|
||||
credentialsStatus: string | null
|
||||
credentialsStatusMessage: string
|
||||
configLoading: boolean
|
||||
manualCheckLoading: boolean
|
||||
submitSitemapLoading: boolean
|
||||
tasksLoading: boolean
|
||||
pagination: any
|
||||
}
|
||||
|
||||
const props = withDefaults(defineProps<Props>(), {
|
||||
systemConfig: null,
|
||||
googleIndexConfig: () => ({}),
|
||||
googleIndexStats: () => ({}),
|
||||
tasks: () => [],
|
||||
credentialsStatus: null,
|
||||
credentialsStatusMessage: '',
|
||||
configLoading: false,
|
||||
manualCheckLoading: false,
|
||||
submitSitemapLoading: false,
|
||||
tasksLoading: false,
|
||||
pagination: () => ({})
|
||||
})
|
||||
|
||||
// Emits
|
||||
const emit = defineEmits<{
|
||||
'update:google-index-config': []
|
||||
'show-verification': []
|
||||
'show-credentials-guide': []
|
||||
'select-credentials-file': []
|
||||
'manual-check-urls': []
|
||||
'refresh-status': []
|
||||
}>()
|
||||
|
||||
// 获取消息组件
|
||||
const message = useMessage()
|
||||
|
||||
// 本地状态
|
||||
const validatingCredentials = ref(false)
|
||||
|
||||
// 计算属性,用于安全地访问凭据文件路径
|
||||
const credentialsFilePath = computed(() => {
|
||||
const path = props.googleIndexConfig?.credentialsFile || ''
|
||||
console.log('Component computed credentialsFilePath:', path)
|
||||
return path
|
||||
})
|
||||
|
||||
|
||||
// 任务表格列
|
||||
const taskColumns = [
|
||||
{
|
||||
title: 'ID',
|
||||
key: 'id',
|
||||
width: 80
|
||||
},
|
||||
{
|
||||
title: '标题',
|
||||
key: 'name',
|
||||
width: 200
|
||||
},
|
||||
{
|
||||
title: '类型',
|
||||
key: 'type',
|
||||
width: 120,
|
||||
render: (row: any) => {
|
||||
const typeMap = {
|
||||
status_check: { text: '状态检查', class: 'bg-blue-100 text-blue-800 dark:bg-blue-900 dark:text-blue-200' },
|
||||
sitemap_submit: { text: '网站地图', class: 'bg-green-100 text-green-800 dark:bg-green-900 dark:text-green-200' },
|
||||
url_indexing: { text: 'URL索引', class: 'bg-yellow-100 text-yellow-800 dark:bg-yellow-900 dark:text-yellow-200' }
|
||||
}
|
||||
const type = typeMap[row.type as keyof typeof typeMap] || { text: row.type, class: 'bg-gray-100 text-gray-800 dark:bg-gray-700 dark:text-gray-200' }
|
||||
return h('span', {
|
||||
class: `px-2 py-1 text-xs font-medium rounded ${type.class}`
|
||||
}, type.text)
|
||||
}
|
||||
},
|
||||
{
|
||||
title: '状态',
|
||||
key: 'status',
|
||||
width: 100,
|
||||
render: (row: any) => {
|
||||
const statusMap = {
|
||||
pending: { text: '待处理', class: 'bg-gray-100 text-gray-800 dark:bg-gray-700 dark:text-gray-200' },
|
||||
running: { text: '运行中', class: 'bg-blue-100 text-blue-800 dark:bg-blue-900 dark:text-blue-200' },
|
||||
completed: { text: '完成', class: 'bg-green-100 text-green-800 dark:bg-green-900 dark:text-green-200' },
|
||||
failed: { text: '失败', class: 'bg-red-100 text-red-800 dark:bg-red-900 dark:text-red-200' }
|
||||
}
|
||||
const status = statusMap[row.status as keyof typeof statusMap] || { text: row.status, class: 'bg-gray-100 text-gray-800 dark:bg-gray-700 dark:text-gray-200' }
|
||||
return h('span', {
|
||||
class: `px-2 py-1 text-xs font-medium rounded ${status.class}`
|
||||
}, status.text)
|
||||
}
|
||||
},
|
||||
{
|
||||
title: '总项目',
|
||||
key: 'totalItems',
|
||||
width: 100
|
||||
},
|
||||
{
|
||||
title: '成功/失败',
|
||||
key: 'progress',
|
||||
width: 120,
|
||||
render: (row: any) => {
|
||||
return h('span', `${row.successful_items} / ${row.failed_items}`)
|
||||
}
|
||||
},
|
||||
{
|
||||
title: '创建时间',
|
||||
key: 'created_at',
|
||||
width: 150,
|
||||
render: (row: any) => {
|
||||
return row.created_at ? new Date(row.created_at).toLocaleString('zh-CN') : 'N/A'
|
||||
}
|
||||
},
|
||||
{
|
||||
title: '操作',
|
||||
key: 'actions',
|
||||
width: 150,
|
||||
render: (row: any) => {
|
||||
return h('div', { class: 'space-x-2' }, [
|
||||
h('button', {
|
||||
class: 'text-blue-600 hover:text-blue-800 dark:text-blue-400 dark:hover:text-blue-300 text-sm',
|
||||
onClick: () => emit('view-task-items', row.id)
|
||||
}, '详情'),
|
||||
h('button', {
|
||||
class: 'text-green-600 hover:text-green-800 dark:text-green-400 dark:hover:text-green-300 text-sm',
|
||||
disabled: row.status !== 'pending' && row.status !== 'running',
|
||||
onClick: () => emit('start-task', row.id)
|
||||
}, '启动')
|
||||
].filter(btn => !btn.props?.disabled))
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
// 验证凭据
|
||||
const validateCredentials = async () => {
|
||||
if (!credentialsFilePath.value) {
|
||||
message.warning('请先上传凭据文件')
|
||||
return
|
||||
}
|
||||
|
||||
validatingCredentials.value = true
|
||||
|
||||
try {
|
||||
const api = useApi()
|
||||
const response = await api.googleIndexApi.validateCredentials({})
|
||||
|
||||
if (response?.valid) {
|
||||
message.success('凭据验证成功')
|
||||
emit('update:google-index-config')
|
||||
} else {
|
||||
message.error('凭据验证失败:' + (response?.message || '凭据无效或权限不足'))
|
||||
}
|
||||
} catch (error: any) {
|
||||
console.error('凭据验证失败:', error)
|
||||
message.error('凭据验证失败: ' + (error?.message || '网络错误'))
|
||||
} finally {
|
||||
validatingCredentials.value = false
|
||||
}
|
||||
}
|
||||
|
||||
// 更新Google索引配置
|
||||
const updateGoogleIndexConfig = async () => {
|
||||
emit('update:google-index-config')
|
||||
}
|
||||
|
||||
// 提交网站地图
|
||||
const submitSitemap = async () => {
|
||||
const siteUrl = props.systemConfig?.site_url || ''
|
||||
if (!siteUrl || siteUrl === 'https://example.com') {
|
||||
message.warning('请先在站点配置中设置正确的站点URL')
|
||||
return
|
||||
}
|
||||
|
||||
const sitemapUrl = siteUrl.endsWith('/') ? siteUrl + 'sitemap.xml' : siteUrl + '/sitemap.xml'
|
||||
|
||||
try {
|
||||
const api = useApi()
|
||||
const response = await api.googleIndexApi.createGoogleIndexTask({
|
||||
title: `网站地图提交任务 - ${new Date().toLocaleString('zh-CN')}`,
|
||||
type: 'sitemap_submit',
|
||||
description: `提交网站地图: ${sitemapUrl}`,
|
||||
SitemapURL: sitemapUrl
|
||||
})
|
||||
if (response) {
|
||||
message.success('网站地图提交任务已创建')
|
||||
emit('refresh-status')
|
||||
}
|
||||
} catch (error: any) {
|
||||
console.error('提交网站地图失败:', error)
|
||||
const errorMsg = error?.response?.data?.message || error?.message || '提交网站地图失败'
|
||||
message.error('提交网站地图失败: ' + errorMsg)
|
||||
}
|
||||
}
|
||||
|
||||
// 获取Google Search Console URL
|
||||
const getSearchConsoleUrl = () => {
|
||||
const siteUrl = props.systemConfig?.site_url || ''
|
||||
if (!siteUrl) {
|
||||
return 'https://search.google.com/search-console'
|
||||
}
|
||||
|
||||
// 格式化URL用于Google Search Console
|
||||
const normalizedUrl = siteUrl.startsWith('http') ? siteUrl : `https://${siteUrl}`
|
||||
return `https://search.google.com/search-console/performance/search-analytics?resource_id=${encodeURIComponent(normalizedUrl)}`
|
||||
}
|
||||
|
||||
// 获取Google Analytics URL
|
||||
const getAnalyticsUrl = () => {
|
||||
const siteUrl = props.systemConfig?.site_url || ''
|
||||
|
||||
// 格式化URL用于Google Analytics
|
||||
const normalizedUrl = siteUrl.startsWith('http') ? siteUrl : `https://${siteUrl}`
|
||||
|
||||
// 跳转到Google Analytics
|
||||
return 'https://analytics.google.com/'
|
||||
}
|
||||
|
||||
// 获取站点URL显示文本
|
||||
const getSiteUrlDisplay = () => {
|
||||
const siteUrl = props.systemConfig?.site_url || ''
|
||||
if (!siteUrl) {
|
||||
return '站点URL未配置'
|
||||
}
|
||||
if (siteUrl === 'https://example.com') {
|
||||
return '请配置正确的站点URL'
|
||||
}
|
||||
return siteUrl
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.tab-content-container {
|
||||
height: calc(100vh - 240px);
|
||||
overflow-y: auto;
|
||||
padding-bottom: 1rem;
|
||||
}
|
||||
</style>
|
||||
199
web/components/Admin/LinkBuildingTab.vue
Normal file
199
web/components/Admin/LinkBuildingTab.vue
Normal file
@@ -0,0 +1,199 @@
|
||||
<template>
|
||||
<div class="tab-content-container">
|
||||
<div class="bg-white dark:bg-gray-800 rounded-lg shadow p-6">
|
||||
<div class="mb-6">
|
||||
<h3 class="text-lg font-semibold text-gray-900 dark:text-white mb-2">外链建设(待开发)</h3>
|
||||
<p class="text-gray-600 dark:text-gray-400">管理和监控外部链接建设情况</p>
|
||||
</div>
|
||||
|
||||
<!-- 外链统计 -->
|
||||
<div class="grid grid-cols-1 md:grid-cols-4 gap-4 mb-6">
|
||||
<div class="bg-blue-50 dark:bg-blue-900/20 rounded-lg p-4">
|
||||
<div class="flex items-center">
|
||||
<div class="p-2 bg-blue-100 dark:bg-blue-900 rounded-lg">
|
||||
<i class="fas fa-link text-blue-600 dark:text-blue-400"></i>
|
||||
</div>
|
||||
<div class="ml-3">
|
||||
<p class="text-sm text-gray-600 dark:text-gray-400">总外链数</p>
|
||||
<p class="text-xl font-bold text-gray-900 dark:text-white">{{ linkStats.total }}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="bg-green-50 dark:bg-green-900/20 rounded-lg p-4">
|
||||
<div class="flex items-center">
|
||||
<div class="p-2 bg-green-100 dark:bg-green-900 rounded-lg">
|
||||
<i class="fas fa-check text-green-600 dark:text-green-400"></i>
|
||||
</div>
|
||||
<div class="ml-3">
|
||||
<p class="text-sm text-gray-600 dark:text-gray-400">有效外链</p>
|
||||
<p class="text-xl font-bold text-gray-900 dark:text-white">{{ linkStats.valid }}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="bg-yellow-50 dark:bg-yellow-900/20 rounded-lg p-4">
|
||||
<div class="flex items-center">
|
||||
<div class="p-2 bg-yellow-100 dark:bg-yellow-900 rounded-lg">
|
||||
<i class="fas fa-clock text-yellow-600 dark:text-yellow-400"></i>
|
||||
</div>
|
||||
<div class="ml-3">
|
||||
<p class="text-sm text-gray-600 dark:text-gray-400">待审核</p>
|
||||
<p class="text-xl font-bold text-gray-900 dark:text-white">{{ linkStats.pending }}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="bg-red-50 dark:bg-red-900/20 rounded-lg p-4">
|
||||
<div class="flex items-center">
|
||||
<div class="p-2 bg-red-100 dark:bg-red-900 rounded-lg">
|
||||
<i class="fas fa-times text-red-600 dark:text-red-400"></i>
|
||||
</div>
|
||||
<div class="ml-3">
|
||||
<p class="text-sm text-gray-600 dark:text-gray-400">失效外链</p>
|
||||
<p class="text-xl font-bold text-gray-900 dark:text-white">{{ linkStats.invalid }}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 外链列表 -->
|
||||
<div class="space-y-4">
|
||||
<div class="flex items-center justify-between">
|
||||
<h4 class="text-lg font-medium text-gray-900 dark:text-white">外链列表</h4>
|
||||
<n-button type="primary" @click="$emit('add-new-link')">
|
||||
<template #icon>
|
||||
<i class="fas fa-plus"></i>
|
||||
</template>
|
||||
添加外链
|
||||
</n-button>
|
||||
</div>
|
||||
|
||||
<n-data-table
|
||||
:columns="linkColumns"
|
||||
:data="linkList"
|
||||
:pagination="pagination"
|
||||
:loading="loading"
|
||||
:bordered="false"
|
||||
striped
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { computed, h } from 'vue'
|
||||
|
||||
// Props
|
||||
interface Props {
|
||||
linkStats: {
|
||||
total: number
|
||||
valid: number
|
||||
pending: number
|
||||
invalid: number
|
||||
}
|
||||
linkList: Array<{
|
||||
id: number
|
||||
url: string
|
||||
title: string
|
||||
status: string
|
||||
domain: string
|
||||
created_at: string
|
||||
}>
|
||||
loading: boolean
|
||||
pagination: any
|
||||
}
|
||||
|
||||
const props = withDefaults(defineProps<Props>(), {
|
||||
linkStats: () => ({
|
||||
total: 0,
|
||||
valid: 0,
|
||||
pending: 0,
|
||||
invalid: 0
|
||||
}),
|
||||
linkList: () => [],
|
||||
loading: false,
|
||||
pagination: () => ({})
|
||||
})
|
||||
|
||||
// Emits
|
||||
const emit = defineEmits<{
|
||||
'add-new-link': []
|
||||
'edit-link': [row: any]
|
||||
'delete-link': [row: any]
|
||||
'load-link-list': [page: number]
|
||||
}>()
|
||||
|
||||
// 表格列配置
|
||||
const linkColumns = [
|
||||
{
|
||||
title: 'URL',
|
||||
key: 'url',
|
||||
width: 300,
|
||||
render: (row: any) => {
|
||||
return h('a', {
|
||||
href: row.url,
|
||||
target: '_blank',
|
||||
class: 'text-blue-600 hover:text-blue-800 dark:text-blue-400 dark:hover:text-blue-300'
|
||||
}, row.url)
|
||||
}
|
||||
},
|
||||
{
|
||||
title: '标题',
|
||||
key: 'title',
|
||||
width: 200
|
||||
},
|
||||
{
|
||||
title: '域名',
|
||||
key: 'domain',
|
||||
width: 150
|
||||
},
|
||||
{
|
||||
title: '状态',
|
||||
key: 'status',
|
||||
width: 100,
|
||||
render: (row: any) => {
|
||||
const statusMap = {
|
||||
valid: { text: '有效', class: 'bg-green-100 text-green-800 dark:bg-green-900 dark:text-green-200' },
|
||||
pending: { text: '待审核', class: 'bg-yellow-100 text-yellow-800 dark:bg-yellow-900 dark:text-yellow-200' },
|
||||
invalid: { text: '失效', class: 'bg-red-100 text-red-800 dark:bg-red-900 dark:text-red-200' }
|
||||
}
|
||||
const status = statusMap[row.status as keyof typeof statusMap]
|
||||
return h('span', {
|
||||
class: `px-2 py-1 text-xs font-medium rounded ${status.class}`
|
||||
}, status.text)
|
||||
}
|
||||
},
|
||||
{
|
||||
title: '创建时间',
|
||||
key: 'created_at',
|
||||
width: 120
|
||||
},
|
||||
{
|
||||
title: '操作',
|
||||
key: 'actions',
|
||||
width: 120,
|
||||
render: (row: any) => {
|
||||
return h('div', { class: 'space-x-2' }, [
|
||||
h('button', {
|
||||
class: 'text-blue-600 hover:text-blue-800 dark:text-blue-400 dark:hover:text-blue-300',
|
||||
onClick: () => emit('edit-link', row)
|
||||
}, '编辑'),
|
||||
h('button', {
|
||||
class: 'text-red-600 hover:text-red-800 dark:text-red-400 dark:hover:text-red-300',
|
||||
onClick: () => emit('delete-link', row)
|
||||
}, '删除')
|
||||
])
|
||||
}
|
||||
}
|
||||
]
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.tab-content-container {
|
||||
height: calc(100vh - 240px);
|
||||
overflow-y: auto;
|
||||
padding-bottom: 1rem;
|
||||
}
|
||||
</style>
|
||||
275
web/components/Admin/SiteSubmitTab.vue
Normal file
275
web/components/Admin/SiteSubmitTab.vue
Normal file
@@ -0,0 +1,275 @@
|
||||
<template>
|
||||
<div class="tab-content-container">
|
||||
<div class="bg-white dark:bg-gray-800 rounded-lg shadow p-6">
|
||||
<div class="mb-6">
|
||||
<h3 class="text-lg font-semibold text-gray-900 dark:text-white mb-2">站点提交(待开发)</h3>
|
||||
<p class="text-gray-600 dark:text-gray-400">向各大搜索引擎提交站点信息</p>
|
||||
</div>
|
||||
|
||||
<!-- 搜索引擎列表 -->
|
||||
<div class="space-y-4">
|
||||
<div class="grid grid-cols-1 md:grid-cols-2 gap-4">
|
||||
<!-- 百度 -->
|
||||
<div class="border border-gray-200 dark:border-gray-700 rounded-lg p-4">
|
||||
<div class="flex items-center justify-between mb-3">
|
||||
<div class="flex items-center space-x-3">
|
||||
<div class="w-8 h-8 bg-blue-500 rounded flex items-center justify-center">
|
||||
<i class="fas fa-search text-white text-sm"></i>
|
||||
</div>
|
||||
<div>
|
||||
<h4 class="font-medium text-gray-900 dark:text-white">百度</h4>
|
||||
<p class="text-xs text-gray-500 dark:text-gray-400">baidu.com</p>
|
||||
</div>
|
||||
</div>
|
||||
<n-button size="small" type="primary" @click="submitToBaidu">
|
||||
<template #icon>
|
||||
<i class="fas fa-paper-plane"></i>
|
||||
</template>
|
||||
提交
|
||||
</n-button>
|
||||
</div>
|
||||
<div class="text-xs text-gray-500 dark:text-gray-400">
|
||||
最后提交时间:{{ lastSubmitTime.baidu || '未提交' }}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 谷歌 -->
|
||||
<div class="border border-gray-200 dark:border-gray-700 rounded-lg p-4">
|
||||
<div class="flex items-center justify-between mb-3">
|
||||
<div class="flex items-center space-x-3">
|
||||
<div class="w-8 h-8 bg-red-500 rounded flex items-center justify-center">
|
||||
<i class="fas fa-globe text-white text-sm"></i>
|
||||
</div>
|
||||
<div>
|
||||
<h4 class="font-medium text-gray-900 dark:text-white">谷歌</h4>
|
||||
<p class="text-xs text-gray-500 dark:text-gray-400">google.com</p>
|
||||
</div>
|
||||
</div>
|
||||
<n-button size="small" type="primary" @click="submitToGoogle">
|
||||
<template #icon>
|
||||
<i class="fas fa-paper-plane"></i>
|
||||
</template>
|
||||
提交
|
||||
</n-button>
|
||||
</div>
|
||||
<div class="text-xs text-gray-500 dark:text-gray-400">
|
||||
最后提交时间:{{ lastSubmitTime.google || '未提交' }}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 必应 -->
|
||||
<div class="border border-gray-200 dark:border-gray-700 rounded-lg p-4">
|
||||
<div class="flex items-center justify-between mb-3">
|
||||
<div class="flex items-center space-x-3">
|
||||
<div class="w-8 h-8 bg-green-500 rounded flex items-center justify-center">
|
||||
<i class="fas fa-search text-white text-sm"></i>
|
||||
</div>
|
||||
<div>
|
||||
<h4 class="font-medium text-gray-900 dark:text-white">必应</h4>
|
||||
<p class="text-xs text-gray-500 dark:text-gray-400">bing.com</p>
|
||||
</div>
|
||||
</div>
|
||||
<n-button size="small" type="primary" @click="submitToBing">
|
||||
<template #icon>
|
||||
<i class="fas fa-paper-plane"></i>
|
||||
</template>
|
||||
提交
|
||||
</n-button>
|
||||
</div>
|
||||
<div class="text-xs text-gray-500 dark:text-gray-400">
|
||||
最后提交时间:{{ lastSubmitTime.bing || '未提交' }}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 搜狗 -->
|
||||
<div class="border border-gray-200 dark:border-gray-700 rounded-lg p-4">
|
||||
<div class="flex items-center justify-between mb-3">
|
||||
<div class="flex items-center space-x-3">
|
||||
<div class="w-8 h-8 bg-orange-500 rounded flex items-center justify-center">
|
||||
<i class="fas fa-search text-white text-sm"></i>
|
||||
</div>
|
||||
<div>
|
||||
<h4 class="font-medium text-gray-900 dark:text-white">搜狗</h4>
|
||||
<p class="text-xs text-gray-500 dark:text-gray-400">sogou.com</p>
|
||||
</div>
|
||||
</div>
|
||||
<n-button size="small" type="primary" @click="submitToSogou">
|
||||
<template #icon>
|
||||
<i class="fas fa-paper-plane"></i>
|
||||
</template>
|
||||
提交
|
||||
</n-button>
|
||||
</div>
|
||||
<div class="text-xs text-gray-500 dark:text-gray-400">
|
||||
最后提交时间:{{ lastSubmitTime.sogou || '未提交' }}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 神马搜索 -->
|
||||
<div class="border border-gray-200 dark:border-gray-700 rounded-lg p-4">
|
||||
<div class="flex items-center justify-between mb-3">
|
||||
<div class="flex items-center space-x-3">
|
||||
<div class="w-8 h-8 bg-purple-500 rounded flex items-center justify-center">
|
||||
<i class="fas fa-mobile-alt text-white text-sm"></i>
|
||||
</div>
|
||||
<div>
|
||||
<h4 class="font-medium text-gray-900 dark:text-white">神马搜索</h4>
|
||||
<p class="text-xs text-gray-500 dark:text-gray-400">sm.cn</p>
|
||||
</div>
|
||||
</div>
|
||||
<n-button size="small" type="primary" @click="submitToShenma">
|
||||
<template #icon>
|
||||
<i class="fas fa-paper-plane"></i>
|
||||
</template>
|
||||
提交
|
||||
</n-button>
|
||||
</div>
|
||||
<div class="text-xs text-gray-500 dark:text-gray-400">
|
||||
最后提交时间:{{ lastSubmitTime.shenma || '未提交' }}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 360搜索 -->
|
||||
<div class="border border-gray-200 dark:border-gray-700 rounded-lg p-4">
|
||||
<div class="flex items-center justify-between mb-3">
|
||||
<div class="flex items-center space-x-3">
|
||||
<div class="w-8 h-8 bg-green-600 rounded flex items-center justify-center">
|
||||
<i class="fas fa-shield-alt text-white text-sm"></i>
|
||||
</div>
|
||||
<div>
|
||||
<h4 class="font-medium text-gray-900 dark:text-white">360搜索</h4>
|
||||
<p class="text-xs text-gray-500 dark:text-gray-400">so.com</p>
|
||||
</div>
|
||||
</div>
|
||||
<n-button size="small" type="primary" @click="submitTo360">
|
||||
<template #icon>
|
||||
<i class="fas fa-paper-plane"></i>
|
||||
</template>
|
||||
提交
|
||||
</n-button>
|
||||
</div>
|
||||
<div class="text-xs text-gray-500 dark:text-gray-400">
|
||||
最后提交时间:{{ lastSubmitTime.so360 || '未提交' }}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 批量提交 -->
|
||||
<div class="mt-6 p-4 bg-blue-50 dark:bg-blue-900/20 rounded-lg border border-blue-200 dark:border-blue-800">
|
||||
<div class="flex items-center justify-between">
|
||||
<div>
|
||||
<h4 class="font-medium text-blue-900 dark:text-blue-100">批量提交</h4>
|
||||
<p class="text-sm text-blue-700 dark:text-blue-300 mt-1">
|
||||
一键提交到所有支持的搜索引擎
|
||||
</p>
|
||||
</div>
|
||||
<n-button type="primary" @click="submitToAll">
|
||||
<template #icon>
|
||||
<i class="fas fa-rocket"></i>
|
||||
</template>
|
||||
批量提交
|
||||
</n-button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { useMessage } from 'naive-ui'
|
||||
import { ref } from 'vue'
|
||||
|
||||
// Props
|
||||
interface Props {
|
||||
lastSubmitTime: {
|
||||
baidu: string
|
||||
google: string
|
||||
bing: string
|
||||
sogou: string
|
||||
shenma: string
|
||||
so360: string
|
||||
}
|
||||
}
|
||||
|
||||
const props = withDefaults(defineProps<Props>(), {
|
||||
lastSubmitTime: () => ({
|
||||
baidu: '',
|
||||
google: '',
|
||||
bing: '',
|
||||
sogou: '',
|
||||
shenma: '',
|
||||
so360: ''
|
||||
})
|
||||
})
|
||||
|
||||
// Emits
|
||||
const emit = defineEmits<{
|
||||
'update:last-submit-time': [engine: string, time: string]
|
||||
}>()
|
||||
|
||||
// 获取消息组件
|
||||
const message = useMessage()
|
||||
|
||||
// 提交到百度
|
||||
const submitToBaidu = () => {
|
||||
const time = new Date().toLocaleString('zh-CN')
|
||||
emit('update:last-submit-time', 'baidu', time)
|
||||
message.success('已提交到百度')
|
||||
}
|
||||
|
||||
// 提交到谷歌
|
||||
const submitToGoogle = () => {
|
||||
const time = new Date().toLocaleString('zh-CN')
|
||||
emit('update:last-submit-time', 'google', time)
|
||||
message.success('已提交到谷歌')
|
||||
}
|
||||
|
||||
// 提交到必应
|
||||
const submitToBing = () => {
|
||||
const time = new Date().toLocaleString('zh-CN')
|
||||
emit('update:last-submit-time', 'bing', time)
|
||||
message.success('已提交到必应')
|
||||
}
|
||||
|
||||
// 提交到搜狗
|
||||
const submitToSogou = () => {
|
||||
const time = new Date().toLocaleString('zh-CN')
|
||||
emit('update:last-submit-time', 'sogou', time)
|
||||
message.success('已提交到搜狗')
|
||||
}
|
||||
|
||||
// 提交到神马搜索
|
||||
const submitToShenma = () => {
|
||||
const time = new Date().toLocaleString('zh-CN')
|
||||
emit('update:last-submit-time', 'shenma', time)
|
||||
message.success('已提交到神马搜索')
|
||||
}
|
||||
|
||||
// 提交到360搜索
|
||||
const submitTo360 = () => {
|
||||
const time = new Date().toLocaleString('zh-CN')
|
||||
emit('update:last-submit-time', 'so360', time)
|
||||
message.success('已提交到360搜索')
|
||||
}
|
||||
|
||||
// 批量提交
|
||||
const submitToAll = () => {
|
||||
const time = new Date().toLocaleString('zh-CN')
|
||||
const engines = ['baidu', 'google', 'bing', 'sogou', 'shenma', 'so360']
|
||||
|
||||
engines.forEach(engine => {
|
||||
emit('update:last-submit-time', engine, time)
|
||||
})
|
||||
|
||||
message.success('已批量提交到所有搜索引擎')
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.tab-content-container {
|
||||
height: calc(100vh - 240px);
|
||||
overflow-y: auto;
|
||||
padding-bottom: 1rem;
|
||||
}
|
||||
</style>
|
||||
226
web/components/Admin/SitemapTab.vue
Normal file
226
web/components/Admin/SitemapTab.vue
Normal file
@@ -0,0 +1,226 @@
|
||||
<template>
|
||||
<div class="tab-content-container">
|
||||
<div class="bg-white dark:bg-gray-800 rounded-lg shadow p-6">
|
||||
<!-- Sitemap配置 -->
|
||||
<div class="mb-6">
|
||||
<h3 class="text-lg font-semibold text-gray-900 dark:text-white mb-2">Sitemap配置</h3>
|
||||
<p class="text-gray-600 dark:text-gray-400">管理网站的Sitemap生成和配置</p>
|
||||
</div>
|
||||
|
||||
<div class="mb-6 p-4 bg-gray-50 dark:bg-gray-700/50 rounded-lg">
|
||||
<div class="flex items-center justify-between mb-4">
|
||||
<div>
|
||||
<h4 class="font-medium text-gray-900 dark:text-white mb-2">自动生成Sitemap</h4>
|
||||
<p class="text-sm text-gray-600 dark:text-gray-400">
|
||||
开启后系统将定期自动生成Sitemap文件
|
||||
</p>
|
||||
</div>
|
||||
<n-switch
|
||||
v-model:value="sitemapConfig.autoGenerate"
|
||||
@update:value="updateSitemapConfig"
|
||||
:loading="configLoading"
|
||||
size="large"
|
||||
>
|
||||
<template #checked>已开启</template>
|
||||
<template #unchecked>已关闭</template>
|
||||
</n-switch>
|
||||
</div>
|
||||
|
||||
<!-- 配置详情 -->
|
||||
<div class="border-t border-gray-200 dark:border-gray-600 pt-4">
|
||||
<div class="grid grid-cols-1 md:grid-cols-2 gap-4">
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">站点URL</label>
|
||||
<n-input
|
||||
:value="systemConfig?.site_url || '站点URL未配置'"
|
||||
:disabled="true"
|
||||
placeholder="请先在站点配置中设置站点URL"
|
||||
>
|
||||
<template #prefix>
|
||||
<i class="fas fa-globe text-gray-400"></i>
|
||||
</template>
|
||||
</n-input>
|
||||
</div>
|
||||
<div class="flex flex-col justify-end">
|
||||
<label class="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-1">最后生成时间</label>
|
||||
<div class="text-sm text-gray-600 dark:text-gray-400">
|
||||
{{ sitemapConfig.lastGenerate || '尚未生成' }}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Sitemap统计 -->
|
||||
<div class="grid grid-cols-1 md:grid-cols-3 gap-4 mb-6">
|
||||
<div class="bg-blue-50 dark:bg-blue-900/20 rounded-lg p-4">
|
||||
<div class="flex items-center">
|
||||
<div class="p-2 bg-blue-100 dark:bg-blue-900 rounded-lg">
|
||||
<i class="fas fa-link text-blue-600 dark:text-blue-400"></i>
|
||||
</div>
|
||||
<div class="ml-3">
|
||||
<p class="text-sm text-gray-600 dark:text-gray-400">资源总数</p>
|
||||
<p class="text-xl font-bold text-gray-900 dark:text-white">{{ sitemapStats.total_resources || 0 }}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="bg-green-50 dark:bg-green-900/20 rounded-lg p-4">
|
||||
<div class="flex items-center">
|
||||
<div class="p-2 bg-green-100 dark:bg-green-900 rounded-lg">
|
||||
<i class="fas fa-sitemap text-green-600 dark:text-green-400"></i>
|
||||
</div>
|
||||
<div class="ml-3">
|
||||
<p class="text-sm text-gray-600 dark:text-gray-400">页面数量</p>
|
||||
<p class="text-xl font-bold text-gray-900 dark:text-white">{{ sitemapStats.total_pages || 0 }}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="bg-purple-50 dark:bg-purple-900/20 rounded-lg p-4">
|
||||
<div class="flex items-center">
|
||||
<div class="p-2 bg-purple-100 dark:bg-purple-900 rounded-lg">
|
||||
<i class="fas fa-history text-purple-600 dark:text-purple-400"></i>
|
||||
</div>
|
||||
<div class="ml-3">
|
||||
<p class="text-sm text-gray-600 dark:text-gray-400">最后更新</p>
|
||||
<p class="text-xl font-bold text-gray-900 dark:text-white">{{ sitemapStats.last_generate || 'N/A' }}</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- 操作按钮 -->
|
||||
<div class="flex flex-wrap gap-3 mb-6">
|
||||
<n-button
|
||||
type="primary"
|
||||
@click="generateSitemap"
|
||||
:loading="isGenerating"
|
||||
size="large"
|
||||
>
|
||||
<template #icon>
|
||||
<i class="fas fa-cog"></i>
|
||||
</template>
|
||||
生成Sitemap
|
||||
</n-button>
|
||||
|
||||
<n-button
|
||||
type="success"
|
||||
@click="viewSitemap"
|
||||
size="large"
|
||||
>
|
||||
<template #icon>
|
||||
<i class="fas fa-external-link-alt"></i>
|
||||
</template>
|
||||
查看Sitemap
|
||||
</n-button>
|
||||
|
||||
<n-button
|
||||
type="info"
|
||||
@click="$emit('refresh-status')"
|
||||
size="large"
|
||||
>
|
||||
<template #icon>
|
||||
<i class="fas fa-sync-alt"></i>
|
||||
</template>
|
||||
刷新状态
|
||||
</n-button>
|
||||
</div>
|
||||
|
||||
<!-- 生成状态 -->
|
||||
<div v-if="generateStatus" class="mb-4 p-3 bg-blue-50 dark:bg-blue-900/20 rounded-lg border border-blue-200 dark:border-blue-800">
|
||||
<div class="flex items-center">
|
||||
<i class="fas fa-info-circle text-blue-500 dark:text-blue-400 mr-2"></i>
|
||||
<span class="text-blue-700 dark:text-blue-300">{{ generateStatus }}</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { useMessage } from 'naive-ui'
|
||||
import { useApi } from '~/composables/useApi'
|
||||
import { ref } from 'vue'
|
||||
|
||||
// Props
|
||||
interface Props {
|
||||
systemConfig?: any
|
||||
sitemapConfig: any
|
||||
sitemapStats: any
|
||||
configLoading: boolean
|
||||
isGenerating: boolean
|
||||
generateStatus: string
|
||||
}
|
||||
|
||||
const props = withDefaults(defineProps<Props>(), {
|
||||
systemConfig: null,
|
||||
sitemapConfig: () => ({}),
|
||||
sitemapStats: () => ({}),
|
||||
configLoading: false,
|
||||
isGenerating: false,
|
||||
generateStatus: ''
|
||||
})
|
||||
|
||||
// Emits
|
||||
const emit = defineEmits<{
|
||||
'update:sitemap-config': [value: boolean]
|
||||
'refresh-status': []
|
||||
}>()
|
||||
|
||||
// 获取消息组件
|
||||
const message = useMessage()
|
||||
|
||||
// 更新Sitemap配置
|
||||
const updateSitemapConfig = async (value: boolean) => {
|
||||
try {
|
||||
const api = useApi()
|
||||
await api.sitemapApi.updateSitemapConfig({
|
||||
autoGenerate: value,
|
||||
lastGenerate: props.sitemapConfig.lastGenerate,
|
||||
lastUpdate: new Date().toISOString()
|
||||
})
|
||||
message.success(value ? '自动生成功能已开启' : '自动生成功能已关闭')
|
||||
} catch (error) {
|
||||
message.error('更新配置失败')
|
||||
// 恢复之前的值
|
||||
props.sitemapConfig.autoGenerate = !value
|
||||
}
|
||||
}
|
||||
|
||||
// 生成Sitemap
|
||||
const generateSitemap = async () => {
|
||||
// 使用已经加载的系统配置
|
||||
const siteUrl = props.systemConfig?.site_url || ''
|
||||
if (!siteUrl) {
|
||||
message.warning('请先在站点配置中设置站点URL,然后再生成Sitemap')
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
const api = useApi()
|
||||
const response = await api.sitemapApi.generateSitemap({ site_url: siteUrl })
|
||||
|
||||
if (response) {
|
||||
message.success(`Sitemap生成任务已启动,使用站点URL: ${siteUrl}`)
|
||||
// 更新统计信息
|
||||
emit('refresh-status')
|
||||
}
|
||||
} catch (error: any) {
|
||||
message.error('Sitemap生成失败')
|
||||
}
|
||||
}
|
||||
|
||||
// 查看Sitemap
|
||||
const viewSitemap = () => {
|
||||
window.open('/sitemap.xml', '_blank')
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.tab-content-container {
|
||||
height: calc(100vh - 240px);
|
||||
overflow-y: auto;
|
||||
padding-bottom: 1rem;
|
||||
}
|
||||
</style>
|
||||
@@ -1,5 +1,6 @@
|
||||
import { useApiFetch } from './useApiFetch'
|
||||
import { useUserStore } from '~/stores/user'
|
||||
import { useGoogleIndexApi } from './useGoogleIndexApi'
|
||||
|
||||
// 统一响应解析函数
|
||||
export const parseApiResponse = <T>(response: any): T => {
|
||||
@@ -447,6 +448,7 @@ export const useApi = () => {
|
||||
apiAccessLogApi: useApiAccessLogApi(),
|
||||
systemLogApi: useSystemLogApi(),
|
||||
wechatApi: useWechatApi(),
|
||||
sitemapApi: useSitemapApi()
|
||||
sitemapApi: useSitemapApi(),
|
||||
googleIndexApi: useGoogleIndexApi()
|
||||
}
|
||||
}
|
||||
245
web/composables/useGoogleIndexApi.ts
Normal file
245
web/composables/useGoogleIndexApi.ts
Normal file
@@ -0,0 +1,245 @@
|
||||
import { useApiFetch } from './useApiFetch'
|
||||
import { parseApiResponse } from './useApi'
|
||||
|
||||
// Google索引配置类型定义
|
||||
export interface GoogleIndexConfig {
|
||||
id?: number
|
||||
group: string
|
||||
key: string
|
||||
value: string
|
||||
type?: string
|
||||
}
|
||||
|
||||
// Google索引任务类型定义
|
||||
export interface GoogleIndexTask {
|
||||
id: number
|
||||
title: string
|
||||
type: string
|
||||
status: string
|
||||
description: string
|
||||
totalItems: number
|
||||
processedItems: number
|
||||
successItems: number
|
||||
failedItems: number
|
||||
indexedURLs: number
|
||||
failedURLs: number
|
||||
errorMessage?: string
|
||||
configID?: number
|
||||
startedAt?: Date
|
||||
completedAt?: Date
|
||||
createdAt: Date
|
||||
updatedAt: Date
|
||||
}
|
||||
|
||||
// Google索引任务项类型定义
|
||||
export interface GoogleIndexTaskItem {
|
||||
id: number
|
||||
taskID: number
|
||||
URL: string
|
||||
status: string
|
||||
indexStatus: string
|
||||
errorMessage?: string
|
||||
inspectResult?: string
|
||||
mobileFriendly: boolean
|
||||
lastCrawled?: Date
|
||||
statusCode: number
|
||||
startedAt?: Date
|
||||
completedAt?: Date
|
||||
createdAt: Date
|
||||
updatedAt: Date
|
||||
}
|
||||
|
||||
// URL状态类型定义
|
||||
export interface GoogleIndexURLStatus {
|
||||
id: number
|
||||
URL: string
|
||||
indexStatus: string
|
||||
lastChecked: Date
|
||||
canonicalURL?: string
|
||||
lastCrawled?: Date
|
||||
changeFreq?: string
|
||||
priority?: number
|
||||
mobileFriendly: boolean
|
||||
robotsBlocked: boolean
|
||||
lastError?: string
|
||||
statusCode: number
|
||||
statusCodeText: string
|
||||
checkCount: number
|
||||
successCount: number
|
||||
failureCount: number
|
||||
createdAt: Date
|
||||
updatedAt: Date
|
||||
}
|
||||
|
||||
// Google索引状态响应类型定义
|
||||
export interface GoogleIndexStatusResponse {
|
||||
enabled: boolean
|
||||
siteURL: string
|
||||
lastCheckTime: Date
|
||||
totalURLs: number
|
||||
indexedURLs: number
|
||||
notIndexedURLs: number
|
||||
errorURLs: number
|
||||
lastSitemapSubmit: Date
|
||||
authValid: boolean
|
||||
}
|
||||
|
||||
// Google索引任务列表响应类型定义
|
||||
export interface GoogleIndexTaskListResponse {
|
||||
tasks: GoogleIndexTask[]
|
||||
total: number
|
||||
page: number
|
||||
pageSize: number
|
||||
totalPages: number
|
||||
}
|
||||
|
||||
// Google索引任务项分页响应类型定义
|
||||
export interface GoogleIndexTaskItemPageResponse {
|
||||
items: GoogleIndexTaskItem[]
|
||||
total: number
|
||||
page: number
|
||||
size: number
|
||||
}
|
||||
|
||||
// Google索引API封装
|
||||
export const useGoogleIndexApi = () => {
|
||||
// 配置管理API
|
||||
const getGoogleIndexConfig = (params?: any) =>
|
||||
useApiFetch('/google-index/config-all', { params }).then(parseApiResponse<GoogleIndexConfig[]>)
|
||||
|
||||
const getGoogleIndexConfigByKey = (key: string) =>
|
||||
useApiFetch(`/google-index/config/${key}`).then(parseApiResponse<GoogleIndexConfig>)
|
||||
|
||||
const updateGoogleIndexConfig = (data: GoogleIndexConfig) =>
|
||||
useApiFetch('/google-index/config', { method: 'POST', body: data }).then(parseApiResponse<GoogleIndexConfig>)
|
||||
|
||||
const deleteGoogleIndexConfig = (key: string) =>
|
||||
useApiFetch(`/google-index/config/${key}`, { method: 'DELETE' }).then(parseApiResponse<boolean>)
|
||||
|
||||
// 任务管理API
|
||||
const getGoogleIndexTasks = (params?: any) =>
|
||||
useApiFetch('/google-index/tasks', { params }).then(parseApiResponse<GoogleIndexTaskListResponse>)
|
||||
|
||||
const getGoogleIndexTask = (id: number) =>
|
||||
useApiFetch(`/google-index/tasks/${id}`).then(parseApiResponse<GoogleIndexTask>)
|
||||
|
||||
const createGoogleIndexTask = (data: any) =>
|
||||
useApiFetch('/google-index/tasks', { method: 'POST', body: data }).then(parseApiResponse<GoogleIndexTask>)
|
||||
|
||||
const startGoogleIndexTask = (id: number) =>
|
||||
useApiFetch(`/google-index/tasks/${id}/start`, { method: 'POST' }).then(parseApiResponse<boolean>)
|
||||
|
||||
const stopGoogleIndexTask = (id: number) =>
|
||||
useApiFetch(`/google-index/tasks/${id}/stop`, { method: 'POST' }).then(parseApiResponse<boolean>)
|
||||
|
||||
const deleteGoogleIndexTask = (id: number) =>
|
||||
useApiFetch(`/google-index/tasks/${id}`, { method: 'DELETE' }).then(parseApiResponse<boolean>)
|
||||
|
||||
// 任务项管理API
|
||||
const getGoogleIndexTaskItems = (taskId: number, params?: any) =>
|
||||
useApiFetch(`/google-index/tasks/${taskId}/items`, { params }).then(parseApiResponse<GoogleIndexTaskItemPageResponse>)
|
||||
|
||||
// URL状态管理API
|
||||
const getGoogleIndexURLStatus = (params?: any) =>
|
||||
useApiFetch('/google-index/urls/status', { params }).then(parseApiResponse<GoogleIndexURLStatus[]>)
|
||||
|
||||
const getGoogleIndexURLStatusByURL = (url: string) =>
|
||||
useApiFetch(`/google-index/urls/status/${encodeURIComponent(url)}`).then(parseApiResponse<GoogleIndexURLStatus>)
|
||||
|
||||
const checkGoogleIndexURLStatus = (data: { urls: string[] }) =>
|
||||
useApiFetch('/google-index/urls/check', { method: 'POST', body: data }).then(parseApiResponse<any>)
|
||||
|
||||
const submitGoogleIndexURL = (data: { urls: string[] }) =>
|
||||
useApiFetch('/google-index/urls/submit', { method: 'POST', body: data }).then(parseApiResponse<any>)
|
||||
|
||||
// 批量操作API
|
||||
const batchSubmitGoogleIndexURLs = (data: { urls: string[], operation: string }) =>
|
||||
useApiFetch('/google-index/batch/submit', { method: 'POST', body: data }).then(parseApiResponse<any>)
|
||||
|
||||
const batchCheckGoogleIndexURLs = (data: { urls: string[], operation: string }) =>
|
||||
useApiFetch('/google-index/batch/check', { method: 'POST', body: data }).then(parseApiResponse<any>)
|
||||
|
||||
// 网站地图提交API
|
||||
const submitGoogleIndexSitemap = (data: { sitemapURL: string }) =>
|
||||
useApiFetch('/google-index/sitemap/submit', { method: 'POST', body: data }).then(parseApiResponse<any>)
|
||||
|
||||
// 状态查询API
|
||||
const getGoogleIndexStatus = () =>
|
||||
useApiFetch('/google-index/status').then(parseApiResponse<GoogleIndexStatusResponse>)
|
||||
|
||||
// 验证凭据API
|
||||
const validateCredentials = (data: { credentialsFile: string }) =>
|
||||
useApiFetch('/google-index/validate-credentials', { method: 'POST', body: data }).then(parseApiResponse<any>)
|
||||
|
||||
// 更新Google索引分组配置API
|
||||
const updateGoogleIndexGroupConfig = (data: GoogleIndexConfig) =>
|
||||
useApiFetch('/google-index/config/update', { method: 'POST', body: data }).then(parseApiResponse<GoogleIndexConfig>)
|
||||
|
||||
// 上传凭据API
|
||||
const uploadCredentials = (file: File) => {
|
||||
const formData = new FormData()
|
||||
formData.append('file', file)
|
||||
return useApiFetch('/google-index/upload-credentials', {
|
||||
method: 'POST',
|
||||
body: formData,
|
||||
headers: {
|
||||
// 注意:此处不应包含Authorization头,因为文件上传通常由use-upload组件处理
|
||||
}
|
||||
}).then(parseApiResponse<any>)
|
||||
}
|
||||
|
||||
// 调度器控制API
|
||||
const startGoogleIndexScheduler = () =>
|
||||
useApiFetch('/google-index/scheduler/start', { method: 'POST' }).then(parseApiResponse<boolean>)
|
||||
|
||||
const stopGoogleIndexScheduler = () =>
|
||||
useApiFetch('/google-index/scheduler/stop', { method: 'POST' }).then(parseApiResponse<boolean>)
|
||||
|
||||
const getGoogleIndexSchedulerStatus = () =>
|
||||
useApiFetch('/google-index/scheduler/status').then(parseApiResponse<any>)
|
||||
|
||||
return {
|
||||
// 配置管理
|
||||
getGoogleIndexConfig,
|
||||
getGoogleIndexConfigByKey,
|
||||
updateGoogleIndexConfig,
|
||||
updateGoogleIndexGroupConfig,
|
||||
deleteGoogleIndexConfig,
|
||||
|
||||
// 凭据验证和上传
|
||||
validateCredentials,
|
||||
uploadCredentials,
|
||||
|
||||
// 任务管理
|
||||
getGoogleIndexTasks,
|
||||
getGoogleIndexTask,
|
||||
createGoogleIndexTask,
|
||||
startGoogleIndexTask,
|
||||
stopGoogleIndexTask,
|
||||
deleteGoogleIndexTask,
|
||||
|
||||
// 任务项管理
|
||||
getGoogleIndexTaskItems,
|
||||
|
||||
// URL状态管理
|
||||
getGoogleIndexURLStatus,
|
||||
getGoogleIndexURLStatusByURL,
|
||||
checkGoogleIndexURLStatus,
|
||||
submitGoogleIndexURL,
|
||||
|
||||
// 批量操作
|
||||
batchSubmitGoogleIndexURLs,
|
||||
batchCheckGoogleIndexURLs,
|
||||
|
||||
// 网站地图提交
|
||||
submitGoogleIndexSitemap,
|
||||
|
||||
// 状态查询
|
||||
getGoogleIndexStatus,
|
||||
|
||||
// 调度器控制
|
||||
startGoogleIndexScheduler,
|
||||
stopGoogleIndexScheduler,
|
||||
getGoogleIndexSchedulerStatus
|
||||
}
|
||||
}
|
||||
@@ -32,6 +32,7 @@
|
||||
import { lightTheme } from 'naive-ui'
|
||||
import { ref, onMounted } from 'vue'
|
||||
|
||||
|
||||
const theme = lightTheme
|
||||
const isDark = ref(false)
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
</n-card>
|
||||
|
||||
<!-- 统计卡片 -->
|
||||
<div class="grid grid-cols-1 md:grid-cols-3 gap-6">
|
||||
<div class="grid grid-cols-1 md:grid-cols-4 gap-6">
|
||||
<!-- 今日资源/总资源数 -->
|
||||
<n-card>
|
||||
<div class="flex items-center">
|
||||
@@ -85,6 +85,27 @@
|
||||
</n-button>
|
||||
</template>
|
||||
</n-card>
|
||||
|
||||
<!-- Google索引统计 -->
|
||||
<n-card>
|
||||
<div class="flex items-center">
|
||||
<div class="p-3 bg-red-100 dark:bg-red-900 rounded-lg">
|
||||
<i class="fab fa-google text-red-600 dark:text-red-400 text-xl"></i>
|
||||
</div>
|
||||
<div class="ml-4">
|
||||
<p class="text-sm font-medium text-gray-600 dark:text-gray-400">已索引URL/总URL</p>
|
||||
<p class="text-2xl font-bold text-gray-900 dark:text-white">{{ googleIndexStats.indexedURLs || 0 }}/{{ googleIndexStats.totalURLs || 0 }}</p>
|
||||
</div>
|
||||
</div>
|
||||
<template #footer>
|
||||
<n-button text type="primary" @click="navigateTo('/admin/seo#google-index')">
|
||||
查看详情
|
||||
<template #icon>
|
||||
<i class="fas fa-arrow-right"></i>
|
||||
</template>
|
||||
</n-button>
|
||||
</template>
|
||||
</n-card>
|
||||
</div>
|
||||
|
||||
<!-- 平台管理 -->
|
||||
@@ -149,6 +170,17 @@ import { useApiFetch } from '~/composables/useApiFetch'
|
||||
import { parseApiResponse } from '~/composables/useApi'
|
||||
import Chart from 'chart.js/auto'
|
||||
|
||||
// Google索引统计
|
||||
const googleIndexStats = ref({
|
||||
totalURLs: 0,
|
||||
indexedURLs: 0,
|
||||
errorURLs: 0,
|
||||
totalTasks: 0,
|
||||
runningTasks: 0,
|
||||
completedTasks: 0,
|
||||
failedTasks: 0
|
||||
})
|
||||
|
||||
// API
|
||||
const statsApi = useStatsApi()
|
||||
const panApi = usePanApi()
|
||||
@@ -370,10 +402,23 @@ watch([weeklyViews, weeklySearches], () => {
|
||||
})
|
||||
})
|
||||
|
||||
// 加载Google索引统计
|
||||
const loadGoogleIndexStats = async () => {
|
||||
try {
|
||||
const response = await useApiFetch('/google-index/status').then(parseApiResponse)
|
||||
if (response && response.data) {
|
||||
googleIndexStats.value = response.data
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('加载Google索引统计失败:', error)
|
||||
}
|
||||
}
|
||||
|
||||
// 组件挂载后初始化图表和数据
|
||||
onMounted(async () => {
|
||||
console.log('组件挂载,开始初始化...')
|
||||
await fetchTrendData()
|
||||
await loadGoogleIndexStats() // 加载Google索引统计
|
||||
console.log('数据获取完成,准备初始化图表...')
|
||||
nextTick(() => {
|
||||
console.log('nextTick执行,开始初始化图表...')
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user