feat(crawler): 영화사 수집(cron) + DB화

- model/company.go: tb_company/tb_company_filmo DB 계층 (Insert/InsertInfo/InsertFilmos/IsExists/ExcludeCodes)
- cron.go: /cron/companyList, /cron/companyInfo 수집 핸들러 (연속오류 가드, company_page.txt 재개)
- constants.go: LAST_PAGE_PATH_COMPANY
- route: /cron/company* 등록

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

config/constants.go

@@ -17,7 +17,8 @@ const (
 	// 로그 파일 경로
 	ERROR_LOG_PATH_KOBIS = "./log/kobis/error.txt"
 	ERROR_LOG_PATH_G2A   = "./log/g2a/error.txt"
-	LAST_PAGE_PATH_KOBIS = "./log/kobis/page.txt"
+	LAST_PAGE_PATH_KOBIS   = "./log/kobis/page.txt"
+	LAST_PAGE_PATH_COMPANY = "./log/kobis/company_page.txt"
 
 	// DB 목록
 	DB_MOVIEW  = "movie"

config/database.json

@@ -14,7 +14,7 @@
         "driver": "mysql",
         "user": "admin",
         "password": "@@20120726KKh",
-        "address": "192.168.0.202:3306",
+        "address": "127.0.0.1:3306",
         "name": "movie"
     }
 }

controller/cron.go

@@ -31,6 +31,7 @@ type Cron struct {
 	MovieInfoModel   model.MovieInfoModel
 	MovieDetailModel model.MovieDetailModel
 	MovieStatsModel  model.MovieStatsModel
+	CompanyModel     model.CompanyModel
 	Kobis            model.Kobis
 	Rest             service.Rest
 }
@@ -528,6 +529,160 @@ func GetKey(c *gin.Context) string {
 	}
 }
 
+/**
+ * 영화진흥위원회 영화사 목록 수집
+ */
+func (this *Cron) CompanyList(c *gin.Context) {
+	var (
+		start                     = time.Now()
+		page                      = GetCompanyPage()
+		perPage                   = 100
+		total, errors, insertRows = 0, 0, 0
+		key                       = GetKey(c)
+	)
+
+	for {
+		var (
+			req        model.SearchCompanyListParams
+			insertData []model.CompanyListInfo
+		)
+
+		req.Key = key
+		req.CurPage = page
+		req.ItemPerPage = perPage
+		data, err := this.Kobis.CompanyListAPI(req)
+
+		if err != nil {
+			SetCompanyPage(page)
+			c.JSON(http.StatusBadRequest, err.Error())
+			return
+		}
+
+		// 더 이상 값이 없다면 중지
+		if data.CompanyListResult.TotCnt <= 0 {
+			break
+		}
+
+		for _, row := range data.CompanyListResult.CompanyList {
+			insertData = append(insertData, row)
+			total++
+		}
+
+		// Insert 는 ON DUPLICATE KEY 로 upsert
+		if len(insertData) > 0 {
+			if err = this.CompanyModel.Insert(insertData); err != nil {
+				errors++
+			} else {
+				insertRows += len(insertData)
+			}
+		}
+
+		fmt.Printf("Company list page %d, total %d, err %d (%.1fs)\n", page, total, errors, time.Since(start).Seconds())
+
+		SetCompanyPage(page)
+		page++
+	}
+
+	c.JSON(http.StatusOK, gin.H{
+		"total":      total,
+		"errors":     errors,
+		"page":       page,
+		"perPage":    perPage,
+		"insertRows": insertRows,
+	})
+}
+
+/**
+ * 영화진흥위원회 영화사 상세 정보 수집 (parts + 필모그래피)
+ */
+func (this *Cron) CompanyInfo(c *gin.Context) {
+	var (
+		start       = time.Now()
+		codes       = this.CompanyModel.CompanyInfoExcludeCodes()
+		total       = 0
+		errors      = 0
+		failedCodes = make([]string, 0)
+		consecutive = 0
+		key         = GetKey(c)
+	)
+
+	for _, companyCd := range codes {
+		var req model.SearchCompanyInfoParams
+		req.Key = key
+		req.CompanyCd = companyCd
+		data, err := this.Kobis.CompanyInfoAPI(req)
+
+		if err != nil {
+			errors++
+			consecutive++
+			failedCodes = append(failedCodes, companyCd)
+
+			// 연속 오류(무효 키, 쿼터 초과 등) 시 중단
+			if consecutive >= 10 {
+				c.JSON(http.StatusBadRequest, gin.H{
+					"message":     "연속 오류 10회로 중단: " + err.Error(),
+					"total":       total,
+					"error":       errors,
+					"failedCodes": failedCodes,
+				})
+				return
+			}
+			continue
+		}
+		consecutive = 0
+
+		info := data.CompanyInfoResult.CompanyInfo
+		if info.CompanyCd == "" {
+			errors++
+			failedCodes = append(failedCodes, companyCd)
+			continue
+		}
+
+		if err = this.CompanyModel.InsertInfo(info); err != nil {
+			errors++
+			continue
+		}
+		if err = this.CompanyModel.InsertFilmos(info.CompanyCd, info.Filmos); err != nil {
+			errors++
+		}
+		total++
+
+		fmt.Printf("Company info %s, total %d, err %d (%.1fs)\n", companyCd, total, errors, time.Since(start).Seconds())
+	}
+
+	c.JSON(http.StatusOK, gin.H{
+		"total":       total,
+		"error":       errors,
+		"failedCodes": failedCodes,
+	})
+}
+
+// 영화사 목록 마지막 호출 Page 저장
+func SetCompanyPage(page int) {
+	data, err := os.Create(config.LAST_PAGE_PATH_COMPANY)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	defer data.Close()
+
+	_, _ = data.WriteString(strconv.FormatInt(int64(page), 10))
+}
+
+// 영화사 목록 마지막 호출 Page 조회
+func GetCompanyPage() int {
+	b, err := os.ReadFile(config.LAST_PAGE_PATH_COMPANY)
+	if err != nil {
+		return 1
+	}
+
+	page, _ := strconv.Atoi(strings.TrimSpace(string(b)))
+	if page == 0 {
+		page = 1
+	}
+	return page
+}
+
 // 마지막 호출 Page 저장
 func SetLastPage(page int) {
 	data, err := os.Create(config.LAST_PAGE_PATH_KOBIS)

model/company.go

@@ -1,5 +1,14 @@
 package model
 
+import (
+	"crawler/config"
+	"crawler/service"
+	"database/sql"
+	"encoding/json"
+	"log"
+	"strings"
+)
+
 // 영화사 목록 검색 변수
 type SearchCompanyListParams struct {
 	Key           string `form:"key" url:"key" binding:"required"`
@@ -60,3 +69,232 @@ type CompanyFilmo struct {
 	MovieNm       string `json:"movieNm"`
 	CompanyPartNm string `json:"companyPartNm"`
 }
+
+// ---- DB 계층 (tb_company / tb_company_filmo) ----
+
+type CompanyModel struct{}
+
+// tb_company 로우
+type CompanyTable struct {
+	CompanyID        int
+	CompanyCd        string
+	CompanyNm        *string
+	CompanyNmEn      *string
+	CeoNm            *string
+	CompanyPartNames *string
+	Parts            *string
+	FilmoNames       *string
+	UpdatedAt        *string
+	CreatedAt        string
+}
+
+func (this *CompanyModel) Total() int {
+	var (
+		db    = service.DB_MOVIEW
+		conn  = db.SQLDB
+		query = "SELECT COUNT(*) FROM tb_company;"
+		total = 0
+	)
+
+	err := conn.QueryRow(query).Scan(&total)
+	if err != nil {
+		db.SetErrorLog(err, query)
+		return total
+	}
+
+	db.SetGeneralLog(config.GL_ACTION_SELECT, query, "select total company")
+	return total
+}
+
+func (this *CompanyModel) IsExists(companyCd string) bool {
+	var (
+		db     = service.DB_MOVIEW
+		conn   = db.SQLDB
+		query  = "SELECT IF(COUNT(*) <= 0, 0, 1) AS `exists` FROM tb_company WHERE company_cd = ?;"
+		exists = false
+	)
+
+	err := conn.QueryRow(query, companyCd).Scan(&exists)
+	if err != nil {
+		db.SetErrorLog(err, query)
+		return exists
+	}
+
+	db.SetGeneralLog(config.GL_ACTION_SELECT, query, "select exists company")
+	return exists
+}
+
+// 목록 API 결과 upsert (company_part_names, filmo_names 채움)
+func (this *CompanyModel) Insert(list []CompanyListInfo) error {
+	if len(list) <= 0 {
+		return nil
+	}
+
+	var (
+		db    = service.DB_MOVIEW
+		conn  = db.SQLDB
+		query = `
+			INSERT INTO tb_company (
+				company_cd, company_nm, company_nm_en, ceo_nm,
+			    company_part_names, filmo_names, updated_at, created_at
+			)
+			VALUES
+		`
+		vals = []interface{}{}
+	)
+
+	for _, row := range list {
+		query += `(?, ?, ?, ?, ?, ?, NULL, NOW()),`
+		vals = append(vals,
+			row.CompanyCd, nullStr(row.CompanyNm), nullStr(row.CompanyNmEn), nullStr(row.CeoNm),
+			nullStr(row.CompanyPartNames), nullStr(row.FilmoNames))
+	}
+
+	query = query[0 : len(query)-1]
+	query += `
+		ON DUPLICATE KEY UPDATE
+			company_nm = VALUES(company_nm), company_nm_en = VALUES(company_nm_en), ceo_nm = VALUES(ceo_nm),
+			company_part_names = VALUES(company_part_names), filmo_names = VALUES(filmo_names), updated_at = NOW();
+	`
+
+	stmt, err := conn.Prepare(query)
+	if err != nil {
+		db.SetErrorLog(err, query)
+		return err
+	}
+	defer stmt.Close()
+
+	_, err = stmt.Exec(vals...)
+	if err != nil {
+		db.SetErrorLog(err, query)
+		return err
+	}
+
+	db.SetGeneralLog(config.GL_ACTION_WRITE, query, "insert company list")
+	return nil
+}
+
+// 상세 API 결과 upsert (parts JSON 채움)
+func (this *CompanyModel) InsertInfo(info CompanyInfo) error {
+	if info.CompanyCd == "" {
+		return nil
+	}
+
+	var (
+		db    = service.DB_MOVIEW
+		conn  = db.SQLDB
+		query = `
+			INSERT INTO tb_company (
+				company_cd, company_nm, company_nm_en, ceo_nm, parts, updated_at, created_at
+			)
+			VALUES (?, ?, ?, ?, ?, NULL, NOW())
+			ON DUPLICATE KEY UPDATE
+				company_nm = VALUES(company_nm), company_nm_en = VALUES(company_nm_en),
+				ceo_nm = VALUES(ceo_nm), parts = VALUES(parts), updated_at = NOW();
+		`
+		parts interface{}
+	)
+
+	if len(info.Parts) > 0 {
+		b, _ := json.Marshal(info.Parts)
+		parts = string(b)
+	} else {
+		parts = nil
+	}
+
+	_, err := conn.Exec(query,
+		info.CompanyCd, nullStr(info.CompanyNm), nullStr(info.CompanyNmEn), nullStr(info.CeoNm), parts)
+	if err != nil {
+		db.SetErrorLog(err, query)
+		return err
+	}
+
+	db.SetGeneralLog(config.GL_ACTION_WRITE, query, "insert company info")
+	return nil
+}
+
+// 필모그래피 upsert (tb_company_filmo)
+func (this *CompanyModel) InsertFilmos(companyCd string, filmos []CompanyFilmo) error {
+	if companyCd == "" || len(filmos) <= 0 {
+		return nil
+	}
+
+	var (
+		db    = service.DB_MOVIEW
+		conn  = db.SQLDB
+		query = `
+			INSERT INTO tb_company_filmo (company_cd, movie_cd, movie_nm, company_part_nm, created_at)
+			VALUES
+		`
+		vals = []interface{}{}
+	)
+
+	for _, f := range filmos {
+		if f.MovieCd == "" {
+			continue
+		}
+		query += `(?, ?, ?, ?, NOW()),`
+		vals = append(vals, companyCd, f.MovieCd, nullStr(f.MovieNm), nullStr(f.CompanyPartNm))
+	}
+
+	if len(vals) == 0 {
+		return nil
+	}
+
+	query = query[0 : len(query)-1]
+	query += `
+		ON DUPLICATE KEY UPDATE movie_nm = VALUES(movie_nm);
+	`
+
+	stmt, err := conn.Prepare(query)
+	if err != nil {
+		db.SetErrorLog(err, query)
+		return err
+	}
+	defer stmt.Close()
+
+	_, err = stmt.Exec(vals...)
+	if err != nil {
+		db.SetErrorLog(err, query)
+		return err
+	}
+
+	db.SetGeneralLog(config.GL_ACTION_WRITE, query, "insert company filmo")
+	return nil
+}
+
+// 상세(parts) 미수집 영화사 코드 조회
+func (this *CompanyModel) CompanyInfoExcludeCodes() []string {
+	var (
+		db    = service.DB_MOVIEW
+		conn  = db.SQLDB
+		query = `SELECT company_cd FROM tb_company WHERE parts IS NULL ORDER BY id ASC;`
+		code  []string
+	)
+
+	rows, err := conn.Query(query)
+	if err != nil && err != sql.ErrNoRows {
+		db.SetErrorLog(err, query)
+		return code
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var cd string
+		if err := rows.Scan(&cd); err != nil {
+			log.Println(err)
+			continue
+		}
+		code = append(code, cd)
+	}
+
+	db.SetGeneralLog(config.GL_ACTION_SELECT, query, "select company_cd exclude codes")
+	return code
+}
+
+func nullStr(s string) interface{} {
+	if strings.TrimSpace(s) == "" {
+		return nil
+	}
+	return s
+}

route/api.go

@@ -71,6 +71,8 @@ func CronRoute(app *gin.Engine) {
 	r.GET("/list", CronController.List)
 	r.GET("/info", CronController.Info)
 	r.GET("/detail", CronController.Detail)
+	r.GET("/companyList", CronController.CompanyList)
+	r.GET("/companyInfo", CronController.CompanyInfo)
 	//r.GET("/stats", CronController.Stats)
 }