GAEアプリのデプロイを実施して、Bigquery上のテーブルに対して毎分処理を実装したいです。
デプロイ時にエラーが出力され原因がよくわからない状況となります。
●auto.sql insert into `fluid-emissary-216806.embulk_test.INPUT_TEST_003` (ID,NUM,STR,VARSTR,DT,TIME0,TIME6,TIME9) select * from `fluid-emissary-216806.embulk_test.INPUT_TEST_001` where Num In (select Num from `fluid-emissary-216806.embulk_test.INPUT_TEST_002` A where not exists(select 1 from `fluid-emissary-216806.embulk_test.INPUT_TEST_003` B where A.NUM=B.NUM)); commit; ●auto.yaml projectid: fluid-emissary-216806 datasetid: embulk_test tableid: INPUT_TEST_003 ●cron.yaml cron: - description: daily run url: /home/dwhtest01/cron schedule: every 1 mins timezone: Asia/Tokyo ●app.yaml runtime: go api_version: go1 handlers: - url: /.* login: admin script: _go_app env_variables: BUCKET_NAME: sample-bucket ●app.go package sample import ( "cloud.google.com/go/bigquery" "cloud.google.com/go/storage" "fmt" "golang.org/x/net/context" "google.golang.org/api/iterator" "google.golang.org/appengine" "google.golang.org/appengine/taskqueue" yaml "gopkg.in/yaml.v2" "io/ioutil" "log" "net/http" "net/url" ) type SqlConfig struct { ProjectID string DatasetID string TableID string Query string } func init() { http.HandleFunc("/home/dwhtest01/cron/task", taskEnqueueHandler) http.HandleFunc("/home/dwhtest01/cron/maketable", makeTableHandler) } func taskEnqueueHandler(w http.ResponseWriter, r *http.Request) { ctx := appengine.NewContext(r) sc, err := getSqlConfig(ctx) if err != nil { log.Fatalf("Cannot get sql config: %v", err) } //taskqueueにpush task := taskqueue.NewPOSTTask("/home/dwhtest01/cron/maketable", url.Values{ "project_id": {sc.ProjectID}, "dataset_id": {sc.DatasetID}, "table_id": {sc.TableID}, "query": {sc.Query}, }) taskqueue.Add(ctx, task, "default") } func getSqlConfig(ctx context.Context) (SqlConfig, error) { bucketname := "" if name, ok := os.LookupEnv("BUCKET_NAME"); ok { bucketname = name } sqlpath := "/home/dwhtest01/cron/auto.sql" confpath := "/home/dwhtest01/cron/auto.yaml" confbuf, err := getFileGCS(ctx, bucketname, confpath) if err != nil { log.Fatalf("Cannot read object: %v", err) } var sc SqlConfig if err := yaml.Unmarshal(confbuf, &sc); err != nil { log.Fatalf("file is not in yaml format: %v", err) } log.Printf("yaml format: %v", sc) querybuf, err := getFileGCS(ctx, bucketname, sqlpath) if err != nil { log.Fatalf("Cannot read object: %v", err) } sc.Query = string(querybuf) return sc, err } func makeTableHandler(w http.ResponseWriter, r *http.Request) { ctx := appengine.NewContext(r) project_id := r.FormValue("project_id") dataset_id := r.FormValue("dataset_id") table_id := r.FormValue("table_id") query := r.FormValue("query") // Creates a client client, err := bigquery.NewClient(ctx, project_id) if err != nil { log.Fatalf("Failed to create client: %v", err) } ds := client.Dataset(dataset_id) // query settings & run q := client.Query(query) q.Dst = ds.Table(table_id) q.CreateDisposition = "CREATE_IF_NEEDED" q.WriteDisposition = "WRITE_TRUNCATE" job, err := q.Run(ctx) if err != nil { // TODO handle error } // Wait until async querying is done. status, err := job.Wait(ctx) if err != nil { // TODO handle error } if err := status.Err(); err != nil { // TODO handle error } it, err := job.Read(ctx) } func getFileGCS(ctx context.Context, bucketname string, filepath string) ([]byte, error) { client, err := storage.NewClient(ctx) if err != nil { log.Fatal(err) } rc, err := client.Bucket(bucketname).Object(filepath).NewReader(ctx) if err != nil { return nil, err } defer rc.Close() data, err := ioutil.ReadAll(rc) if err != nil { log.Fatal(err) } return data, err }
※エラー
dwhtest01@cloudshell:~/cron (fluid-emissary-216806)$ gcloud app deploy app.yaml ERROR: gcloud crashed (BadStatusLine): '' If you would like to report this issue, please run the following command: gcloud feedback To check gcloud for common problems, please run the following command: gcloud info --run-diagnostics dwhtest01@cloudshell:~/cron (fluid-emissary-216806)$ gcloud info --run-diagnostics Network diagnostic detects and fixes local network connection issues. Checking network connection...done. Reachability Check passed. Network diagnostic passed (1/1 checks passed). Property diagnostic detects issues that may be caused by properties. Checking hidden properties...done. ERROR: Hidden Property Check failed. The following hidden properties have been set: [compute/gce_metadata_read_timeout_sec] Properties files User: /tmp/tmp.xFf1q20vjB/configurations/config_cloudshell-397 Installation: /google/google-cloud-sdk/properties ERROR: Property diagnostic failed (0/1 checks passed). dwhtest01@cloudshell:~/cron (fluid-emissary-216806)$
※2019/01/10
dwhtest01@cloudshell:~/cron (fluid-emissary-216806)$ gcloud app deploy app.yaml ERROR: (gcloud.app.deploy) Staging command [/google/google-cloud-sdk/platform/google_appengine/go-app-stager /home/dwhtest01/cron/app.yaml /home/dwhtest01/cron /tmp/tmpd9Qyvc/tmpN8ZvfG] failed with return code [1]. ------------------------------------ STDOUT ------------------------------------ ------------------------------------ STDERR ------------------------------------ 2019/01/10 08:46:45 staging for go1.9 2019/01/10 08:46:45 Staging Standard app: failed analyzing /home/dwhtest01/cron: cannot find package "google.golang.org/api/iterator" in any of: ($GOROOT not set) /home/dwhtest01/gopath/src/google.golang.org/api/iterator (from $GOPATH) /google/gopath/src/google.golang.org/api/iterator GOPATH: /home/dwhtest01/gopath:/google/gopath -------------------------------------------------------------------------------- dwhtest01@cloudshell:~/cron (fluid-emissary-216806)$
※2019/1/11 ■Go用Google API拡張モジュールインストール後にリトライ、再びエラー発生
$ go get -u github.com/googleapis/gax-go $ go get -u google.golang.org/api/bigquery/v2 $ go get -u gopkg.in/yaml.v2 $ go get -u go.opencensus.io/trace $ go get -u fmt $ go get -u google.golang.org/api/iterator dwhtest01@cloudshell:~/cron (fluid-emissary-216806)$ gcloud app deploy app.yaml Services to deploy: descriptor: [/home/dwhtest01/cron/app.yaml] source: [/home/dwhtest01/cron] target project: [fluid-emissary-216806] target service: [default] target version: [20190111t105305] target url: [https://fluid-emissary-216806.appspot.com] Do you want to continue (Y/n)? y Beginning deployment of service [default]... ?????????????????????????????????????????????????????????????? ?? Uploading 1 file to Google Cloud Storage ?? ?????????????????????????????????????????????????????????????? File upload done. Updating service [default]...failed. ERROR: (gcloud.app.deploy) Error Response: [9] Deployment contains files that cannot be compiled: Compile failed: /work_dir/app.go:50:17: undefined: os 2019/01/10 17:53:23 go-app-builder: build timing: 108×compile (24.98s total), 0×link (0s total) 2019/01/10 17:53:23 go-app-builder: failed running compile: exit status 2 dwhtest01@cloudshell:~/cron (fluid-emissary-216806)$
※※2019/1/21 ■app.goを編集し再実行
dwhtest01@cloudshell:~/cron (fluid-emissary-216806)$ gcloud app deploy app.yaml Services to deploy: descriptor: [/home/dwhtest01/cron/app.yaml] source: [/home/dwhtest01/cron] target project: [fluid-emissary-216806] target service: [default] target version: [20190121t160337] target url: [https://fluid-emissary-216806.appspot.com] Do you want to continue (Y/n)? y Beginning deployment of service [default]... ?????????????????????????????????????????????????????????????? ?? Uploading 1 file to Google Cloud Storage ?? ?????????????????????????????????????????????????????????????? File upload done. Updating service [default]...failed. ERROR: (gcloud.app.deploy) Error Response: [9] Deployment contains files that cannot be compiled: Compile failed: /work_dir/app.go:110:22: it declared and not used 2019/01/20 23:03:57 go-app-builder: build timing: 108×compile (24.657s total), 0×link (0s total) 2019/01/20 23:03:57 go-app-builder: failed running compile: exit status 2
回答1件
あなたの回答
tips
プレビュー
バッドをするには、ログインかつ
こちらの条件を満たす必要があります。
2019/01/21 05:13
2019/01/21 06:19
2019/01/21 07:08
2019/01/23 06:21
2019/02/07 05:07