processing-server/render.yaml at dev · HUFS-Capstone-Project/processing-server · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
services:
  - type: pserv
    name: processing-private-service
    runtime: docker
    region: singapore
    dockerfilePath: ./Dockerfile
    dockerCommand: uvicorn app.main:app --host 0.0.0.0 --port $PORT
    autoDeploy: true
    envVars:
      - key: ENVIRONMENT
        value: production
      - key: INTERNAL_API_KEY
        sync: false
      - key: DATABASE_URL
        sync: false
      - key: PROCESSING_SCHEMA
        value: processing
      - key: QUEUE_NAMESPACE
        value: processing:jobs
      - key: QUEUE_REDIS_URL
        fromService:
          type: keyvalue
          name: processing-queue
          property: connectionString

  - type: worker
    name: processing-worker
    runtime: docker
    region: singapore
    dockerfilePath: ./Dockerfile
    dockerCommand: python -m app.worker.runner
    autoDeploy: true
    envVars:
      - key: ENVIRONMENT
        value: production
      - key: DATABASE_URL
        sync: false
      - key: PROCESSING_SCHEMA
        value: processing
      - key: QUEUE_NAMESPACE
        value: processing:jobs
      - key: QUEUE_REDIS_URL
        fromService:
          type: keyvalue
          name: processing-queue
          property: connectionString
      - key: QUEUE_POP_TIMEOUT_SECONDS
        value: "5"
      - key: QUEUE_PROMOTE_BATCH_SIZE
        value: "50"
      - key: WORKER_MAX_ATTEMPTS
        value: "3"
      - key: WORKER_RETRY_BASE_SECONDS
        value: "10"
      - key: WORKER_RETRY_MAX_SECONDS
        value: "300"
      - key: WORKER_IDLE_SLEEP_SECONDS
        value: "1.0"
      - key: WORKER_PREWARM_BROWSER
        value: "true"
      - key: WORKER_PREWARM_TIMEOUT_SECONDS
        value: "15"
      - key: CRAWLER_TIMEOUT
        value: "45"
      - key: INSTAGRAM_NAVIGATION_TIMEOUT
        value: "20"
      - key: INSTAGRAM_OG_WAIT_TIMEOUT_MS
        value: "5000"
      - key: INSTAGRAM_BLOCK_RESOURCE_TYPES
        value: "image,font,media"
      - key: CRAWLER_BROWSER_REUSE_ENABLED
        value: "true"
      - key: CRAWLER_RECOVER_ON_BROWSER_CRASH
        value: "true"
      - key: CRAWLER_HARD_TIMEOUT_MARGIN_SECONDS
        value: "10"
      - key: KAKAO_TIMEOUT_SECONDS
        value: "10"
      - key: HF_EXTRACTION_TIMEOUT_SECONDS
        value: "30"
      - key: HF_EXTRACTION_MAX_NEW_TOKENS
        value: "1024"
      - key: HF_EXTRACTION_MAX_ATTEMPTS
        value: "3"
      - key: HF_EXTRACTION_RETRY_BASE_SECONDS
        value: "2"
      - key: HF_EXTRACTION_RETRY_BACKOFF_MULTIPLIER
        value: "2.0"
      - key: EXTRACTION_FAILURE_RETRY_ENABLED
        value: "true"
      - key: WORKER_METRICS_LOG_INTERVAL_SECONDS
        value: "60"
      - key: PLAYWRIGHT_NO_SANDBOX
        value: "true"
      - key: PLAYWRIGHT_DISABLE_DEV_SHM_USAGE
        value: "true"

  - type: worker
    name: business-hours-worker
    runtime: docker
    region: singapore
    dockerfilePath: ./Dockerfile
    dockerCommand: python -m app.worker.business_hours_runner
    autoDeploy: true
    envVars:
      - key: ENVIRONMENT
        value: production
      - key: DATABASE_URL
        sync: false
      - key: PROCESSING_SCHEMA
        value: processing
      - key: QUEUE_REDIS_URL
        fromService:
          type: keyvalue
          name: processing-queue
          property: connectionString
      - key: BUSINESS_HOURS_QUEUE_NAMESPACE
        value: processing:business-hours
      - key: BUSINESS_HOURS_QUEUE_POP_TIMEOUT_SECONDS
        value: "5"
      - key: BUSINESS_HOURS_WORKER_CONCURRENCY
        value: "1"
      - key: BUSINESS_HOURS_WORKER_IDLE_SLEEP_SECONDS
        value: "1.0"
      - key: BUSINESS_HOURS_CRAWL_TIMEOUT_SECONDS
        value: "30"
      - key: BUSINESS_HOURS_CRAWL_NAVIGATION_TIMEOUT_SECONDS
        value: "15"
      - key: BUSINESS_HOURS_CRAWL_SELECTOR_WAIT_TIMEOUT_MS
        value: "5000"
      - key: BUSINESS_HOURS_CRAWL_FALLBACK_WAIT_TIMEOUT_MS
        value: "2000"
      - key: BUSINESS_HOURS_CRAWL_NETWORKIDLE_ENABLED
        value: "false"
      - key: BUSINESS_HOURS_CRAWL_NETWORKIDLE_TIMEOUT_MS
        value: "5000"
      - key: BUSINESS_HOURS_CRAWL_MAX_ATTEMPTS
        value: "2"
      - key: BUSINESS_HOURS_CRAWL_RETRY_BASE_MS
        value: "1000"
      - key: BUSINESS_HOURS_FETCHING_STALE_TIMEOUT_SECONDS
        value: "900"
      - key: BUSINESS_HOURS_SUCCESS_TTL_SECONDS
        value: "1209600"
      - key: BUSINESS_HOURS_NOT_FOUND_TTL_SECONDS
        value: "259200"
      - key: BUSINESS_HOURS_CRAWL_FAILED_TTL_SECONDS
        value: "21600"
      - key: BUSINESS_HOURS_PARSE_FAILED_TTL_SECONDS
        value: "86400"
      - key: BUSINESS_HOURS_ENQUEUE_FAILED_TTL_SECONDS
        value: "600"
      - key: PLAYWRIGHT_NO_SANDBOX
        value: "true"
      - key: PLAYWRIGHT_DISABLE_DEV_SHM_USAGE
        value: "true"

  - type: keyvalue
    name: processing-queue
    region: singapore
    ipAllowList: []