90CNServer/server/internal/platform/datatable/generator.go

package datatable

import (
	"archive/zip"
	"bytes"
	"context"
	"crypto/sha256"
	"encoding/csv"
	"encoding/hex"
	"encoding/json"
	"encoding/xml"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"time"
)

var errXLSXPartNotFound = errors.New("xlsx part not found")

type GenerateOptions struct {
	Input       string `json:"input"`
	Output      string `json:"output"`
	Format      string `json:"format,omitempty"`
	Version     string `json:"version,omitempty"`
	ManifestOut string `json:"manifest_out,omitempty"`
	Pretty      bool   `json:"pretty,omitempty"`
}

type GeneratedTable struct {
	Name     string `json:"name"`
	Source   string `json:"source"`
	Output   string `json:"output"`
	Rows     int    `json:"rows"`
	Bytes    int64  `json:"bytes"`
	Checksum string `json:"checksum"`
}

type GenerateReport struct {
	Input       string           `json:"input"`
	Output      string           `json:"output"`
	Version     string           `json:"version,omitempty"`
	Tables      []GeneratedTable `json:"tables"`
	Manifest    Manifest         `json:"manifest"`
	GeneratedAt time.Time        `json:"generated_at"`
}

func GenerateTables(ctx context.Context, options GenerateOptions) (GenerateReport, error) {
	if err := contextErr(ctx); err != nil {
		return GenerateReport{}, err
	}
	options.Input = strings.TrimSpace(options.Input)
	options.Output = strings.TrimSpace(options.Output)
	options.Format = strings.ToLower(strings.TrimSpace(options.Format))
	options.Version = strings.TrimSpace(options.Version)
	options.ManifestOut = strings.TrimSpace(options.ManifestOut)
	if options.Input == "" {
		return GenerateReport{}, ErrDirectoryRequired
	}
	if options.Output == "" {
		return GenerateReport{}, fmt.Errorf("data table output is required")
	}
	info, err := os.Stat(options.Input)
	if err != nil {
		return GenerateReport{}, err
	}
	sources, err := generatorSources(options.Input, info)
	if err != nil {
		return GenerateReport{}, err
	}
	if len(sources) == 0 {
		return GenerateReport{}, fmt.Errorf("no data table source files found in %s", options.Input)
	}
	if err := os.MkdirAll(options.Output, 0o750); err != nil {
		return GenerateReport{}, err
	}
	report := GenerateReport{
		Input:       options.Input,
		Output:      options.Output,
		Version:     options.Version,
		GeneratedAt: time.Now().UTC(),
	}
	tableSources := make(map[string]string, len(sources))
	for _, source := range sources {
		if err := contextErr(ctx); err != nil {
			return GenerateReport{}, err
		}
		tableName := tableNameFromFile(source)
		tableKey := strings.ToLower(tableName)
		if previous, ok := tableSources[tableKey]; ok {
			return GenerateReport{}, fmt.Errorf("duplicate generated data table name %q from %s and %s", tableName, previous, source)
		}
		tableSources[tableKey] = source
		rows, err := loadGeneratedRows(source, options.Format)
		if err != nil {
			return GenerateReport{}, fmt.Errorf("generate table %s: %w", source, err)
		}
		data, err := marshalGeneratedRows(rows, options.Pretty)
		if err != nil {
			return GenerateReport{}, err
		}
		outPath := filepath.Join(options.Output, tableName+".json")
		if err := os.WriteFile(outPath, data, 0o600); err != nil {
			return GenerateReport{}, err
		}
		checksum := sha256.Sum256(data)
		report.Tables = append(report.Tables, GeneratedTable{
			Name:     tableName,
			Source:   source,
			Output:   outPath,
			Rows:     len(rows),
			Bytes:    int64(len(data)),
			Checksum: hex.EncodeToString(checksum[:]),
		})
	}
	sort.Slice(report.Tables, func(i, j int) bool { return report.Tables[i].Name < report.Tables[j].Name })
	registry := NewRegistry(options.Output, options.Version)
	if err := registry.Load(ctx); err != nil {
		return GenerateReport{}, err
	}
	report.Manifest = registry.Manifest()
	if options.ManifestOut != "" {
		manifestData, err := json.MarshalIndent(report.Manifest, "", "  ")
		if err != nil {
			return GenerateReport{}, err
		}
		if err := os.MkdirAll(filepath.Dir(options.ManifestOut), 0o750); err != nil && filepath.Dir(options.ManifestOut) != "." {
			return GenerateReport{}, err
		}
		if err := os.WriteFile(options.ManifestOut, append(manifestData, '\n'), 0o600); err != nil {
			return GenerateReport{}, err
		}
	}
	return report, nil
}

func generatorSources(input string, info os.FileInfo) ([]string, error) {
	if !info.IsDir() {
		if genExtSupported(input) {
			return []string{input}, nil
		}
		return nil, fmt.Errorf("unsupported data table source %s", input)
	}
	var sources []string
	if err := filepath.WalkDir(input, func(path string, entry os.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if entry.IsDir() {
			return nil
		}
		if strings.HasPrefix(filepath.Base(path), ".") {
			return nil
		}
		if genExtSupported(path) {
			sources = append(sources, path)
		}
		return nil
	}); err != nil {
		return nil, err
	}
	sort.Strings(sources)
	return sources, nil
}

func genExtSupported(path string) bool {
	switch strings.ToLower(filepath.Ext(path)) {
	case ".json", ".csv", ".xlsx":
		return true
	default:
		return false
	}
}

func loadGeneratedRows(path, format string) ([]map[string]any, error) {
	if format == "" || format == "auto" {
		format = strings.TrimPrefix(strings.ToLower(filepath.Ext(path)), ".")
	}
	switch format {
	case "json":
		return loadGeneratedJSON(path)
	case "csv":
		return loadGeneratedCSV(path)
	case "xlsx":
		return loadGeneratedXLSX(path)
	default:
		return nil, fmt.Errorf("unsupported data table format %q", format)
	}
}

func loadGeneratedJSON(path string) ([]map[string]any, error) {
	data, err := os.ReadFile(path) //nolint:gosec // G304：路径来自框架配置、仓库扫描或测试临时目录，调用点负责限定输入范围。
	if err != nil {
		return nil, err
	}
	var rows []map[string]any
	if err := json.Unmarshal(data, &rows); err == nil {
		return rows, nil
	}
	var wrapped struct {
		Items []map[string]any `json:"items"`
	}
	if err := json.Unmarshal(data, &wrapped); err != nil {
		return nil, err
	}
	if wrapped.Items == nil {
		return nil, fmt.Errorf("json table must be an array or object with items")
	}
	return wrapped.Items, nil
}

func loadGeneratedCSV(path string) ([]map[string]any, error) {
	file, err := os.Open(path) //nolint:gosec // G304：路径来自框架配置、仓库扫描或测试临时目录，调用点负责限定输入范围。
	if err != nil {
		return nil, err
	}
	defer func() { _ = file.Close() }()
	reader := csv.NewReader(file)
	reader.TrimLeadingSpace = true
	reader.FieldsPerRecord = -1
	records, err := reader.ReadAll()
	if err != nil {
		return nil, err
	}
	return recordsToRows(records)
}

func loadGeneratedXLSX(path string) ([]map[string]any, error) {
	reader, err := zip.OpenReader(path)
	if err != nil {
		return nil, err
	}
	defer func() { _ = reader.Close() }()
	shared, err := readXLSXStrings(&reader.Reader)
	if err != nil {
		return nil, err
	}
	sheet, err := readZipFile(&reader.Reader, "xl/worksheets/sheet1.xml")
	if err != nil {
		return nil, err
	}
	records, err := parseXLSXSheet(sheet, shared)
	if err != nil {
		return nil, err
	}
	return recordsToRows(records)
}

func recordsToRows(records [][]string) ([]map[string]any, error) {
	if len(records) == 0 {
		return nil, nil
	}
	headers := normalizeHeaders(records[0])
	if len(headers) == 0 {
		return nil, fmt.Errorf("data table header row is empty")
	}
	rows := make([]map[string]any, 0, len(records)-1)
	for _, record := range records[1:] {
		if emptyRecord(record) {
			continue
		}
		row := make(map[string]any, len(headers))
		for idx, header := range headers {
			value := ""
			if idx < len(record) {
				value = strings.TrimSpace(record[idx])
			}
			row[header] = inferGeneratedValue(value)
		}
		rows = append(rows, row)
	}
	return rows, nil
}

func normalizeHeaders(headers []string) []string {
	out := make([]string, 0, len(headers))
	seen := make(map[string]struct{}, len(headers))
	for idx, header := range headers {
		header = strings.TrimSpace(header)
		if header == "" {
			header = fmt.Sprintf("col_%d", idx+1)
		}
		base := header
		for suffix := 2; ; suffix++ {
			if _, ok := seen[header]; !ok {
				break
			}
			header = fmt.Sprintf("%s_%d", base, suffix)
		}
		seen[header] = struct{}{}
		out = append(out, header)
	}
	return out
}

func emptyRecord(record []string) bool {
	for _, value := range record {
		if strings.TrimSpace(value) != "" {
			return false
		}
	}
	return true
}

func inferGeneratedValue(value string) any {
	value = strings.TrimSpace(value)
	if value == "" {
		return ""
	}
	switch strings.ToLower(value) {
	case "true", "false":
		parsed, _ := strconv.ParseBool(strings.ToLower(value))
		return parsed
	}
	if parsed, err := strconv.ParseInt(value, 10, 64); err == nil {
		return parsed
	}
	if parsed, err := strconv.ParseFloat(value, 64); err == nil {
		return parsed
	}
	return value
}

func marshalGeneratedRows(rows []map[string]any, pretty bool) ([]byte, error) {
	var data []byte
	var err error
	if pretty {
		data, err = json.MarshalIndent(rows, "", "  ")
	} else {
		data, err = json.Marshal(rows)
	}
	if err != nil {
		return nil, err
	}
	return append(data, '\n'), nil
}

func tableNameFromFile(path string) string {
	name := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))
	name = strings.TrimSpace(name)
	if name == "" {
		return "table"
	}
	return name
}

func readZipFile(reader *zip.Reader, name string) ([]byte, error) {
	for _, file := range reader.File {
		if filepath.ToSlash(file.Name) != name {
			continue
		}
		rc, err := file.Open()
		if err != nil {
			return nil, err
		}
		defer func() { _ = rc.Close() }()
		return io.ReadAll(rc)
	}
	return nil, fmt.Errorf("%w: %s", errXLSXPartNotFound, name)
}

func readXLSXStrings(reader *zip.Reader) ([]string, error) {
	data, err := readZipFile(reader, "xl/sharedStrings.xml")
	if err != nil {
		if errors.Is(err, errXLSXPartNotFound) {
			return nil, nil
		}
		return nil, err
	}
	decoder := xml.NewDecoder(bytes.NewReader(data))
	var values []string
	var current strings.Builder
	inText := false
	for {
		token, err := decoder.Token()
		if errors.Is(err, io.EOF) {
			break
		}
		if err != nil {
			return nil, err
		}
		switch t := token.(type) {
		case xml.StartElement:
			if t.Name.Local == "si" {
				current.Reset()
			}
			if t.Name.Local == "t" {
				inText = true
			}
		case xml.EndElement:
			if t.Name.Local == "t" {
				inText = false
			}
			if t.Name.Local == "si" {
				values = append(values, current.String())
			}
		case xml.CharData:
			if inText {
				current.Write([]byte(t))
			}
		}
	}
	return values, nil
}

func parseXLSXSheet(data []byte, shared []string) ([][]string, error) {
	decoder := xml.NewDecoder(bytes.NewReader(data))
	var records [][]string
	var row []string
	var cellRef string
	var cellType string
	var value strings.Builder
	inValue := false
	inInlineText := false
	for {
		token, err := decoder.Token()
		if errors.Is(err, io.EOF) {
			break
		}
		if err != nil {
			return nil, err
		}
		switch t := token.(type) {
		case xml.StartElement:
			switch t.Name.Local {
			case "row":
				row = nil
			case "c":
				cellRef = ""
				cellType = ""
				value.Reset()
				for _, attr := range t.Attr {
					switch attr.Name.Local {
					case "r":
						cellRef = attr.Value
					case "t":
						cellType = attr.Value
					}
				}
			case "v":
				inValue = true
			case "t":
				if cellType == "inlineStr" {
					inInlineText = true
				}
			}
		case xml.EndElement:
			switch t.Name.Local {
			case "v":
				inValue = false
			case "t":
				inInlineText = false
			case "c":
				text := decodeXLSXCell(value.String(), cellType, shared)
				idx := xlsxCellIndex(cellRef)
				if idx < 0 {
					idx = len(row)
				}
				for len(row) <= idx {
					row = append(row, "")
				}
				row[idx] = text
			case "row":
				records = append(records, append([]string(nil), row...))
			}
		case xml.CharData:
			if inValue || inInlineText {
				value.Write([]byte(t))
			}
		}
	}
	return records, nil
}

func decodeXLSXCell(raw, cellType string, shared []string) string {
	raw = strings.TrimSpace(raw)
	if cellType != "s" {
		return raw
	}
	idx, err := strconv.Atoi(raw)
	if err != nil || idx < 0 || idx >= len(shared) {
		return raw
	}
	return shared[idx]
}

func xlsxCellIndex(ref string) int {
	ref = strings.TrimSpace(ref)
	if ref == "" {
		return -1
	}
	idx := 0
	found := false
	for _, r := range ref {
		if r >= 'A' && r <= 'Z' {
			idx = idx*26 + int(r-'A'+1)
			found = true
			continue
		}
		if r >= 'a' && r <= 'z' {
			idx = idx*26 + int(r-'a'+1)
			found = true
			continue
		}
		break
	}
	if !found {
		return -1
	}
	return idx - 1
}