cloudflare/GCS-Logshare-Setup-Script
Publicmirrored fromhttps://github.com/cloudflare/GCS-Logshare-Setup-Script
gcs-initialize.sh
97lines · modecode
Updated logshare-cli cmd line option to match spec. Updated a few things in the readme such as removing /received because it's the default endpoint now. Updated main.sh script output73222c3
8 years ago
| 1 | #! /bin/bash |
| 2 | |
| 3 | if [ ! -e /root/.secondboot ]; then |
| 4 | logger "Creating dot file to prevent multiple executions" |
| 5 | touch /root/.secondboot |
| 6 | |
| 7 | # Run Dependencies |
| 8 | logger "Installing dependencies: jq git zip golang1.8" |
| 9 | add-apt-repository ppa:gophers/archive -y |
| 10 | apt-get update |
| 11 | apt-get install -y jq git zip golang-1.8 |
| 12 | |
| 13 | mkdir /root/go |
| 14 | export GOPATH=/root/go |
| 15 | export PATH=$PATH:/usr/lib/go-1.8/bin |
| 16 | export RANDOMVALUE=`curl http://metadata.google.internal/computeMetadata/v1/instance/attributes/RANDOMVALUE -H "Metadata-Flavor: Google"` |
| 17 | |
| 18 | # Export Go Environment Variables |
| 19 | echo -e "export GOPATH=/root/go\nexport PATH="$PATH:/usr/lib/go-1.8/bin"" >> /root/.bashrc |
| 20 | |
| 21 | # Get logshare-cli |
| 22 | logger "Cloning Cloudflare Logshare" |
| 23 | /usr/lib/go-1.8/bin/go get github.com/cloudflare/logshare/... |
| 24 | go get github.com/cloudflare/logshare/... |
| 25 | |
| 26 | # Copy GCS config files |
| 27 | logger "Copying Config files from bucket" |
| 28 | gsutil cp gs://`curl http://metadata.google.internal/computeMetadata/v1/instance/attributes/CONFIGBUCKET -H "Metadata-Flavor: Google"`/config.json /root/ |
| 29 | |
| 30 | # Export config.json into environment variables |
| 31 | logger "Exporting environment variables from config file" |
| 32 | echo "export APIKEY=`jq -r .cloudflare_api_key /root/config.json`" >> /root/.bashrc |
| 33 | export APIKEY=`jq -r .cloudflare_api_key /root/config.json` |
| 34 | |
| 35 | echo "export APIEMAIL=`jq -r .cloudflare_api_email /root/config.json`" >> /root/.bashrc |
| 36 | export APIEMAIL=`jq -r .cloudflare_api_email /root/config.json` |
| 37 | |
| 38 | echo "export ZONENAME=`jq -r .zone_name /root/config.json`" >> /root/.bashrc |
| 39 | export ZONENAME=`jq -r .zone_name /root/config.json` |
| 40 | |
| 41 | # Source bashrc |
| 42 | source /root/.bashrc |
| 43 | |
| 44 | |
| 45 | # Create one-time fields.txt - outputs all available fields as of creation - may need to be updated in future |
| 46 | logger "Caching most recent fields from ELS /received endpoint" |
| 47 | /root/go/bin/logshare-cli --api-key=$APIKEY --api-email=$APIEMAIL --zone-name $ZONENAME --list-fields 2> /dev/null | jq -r '. | keys_unsorted | @csv' | tr -d '"' > /root/fields.txt |
| 48 | |
| 49 | # Create cron-script.sh |
| 50 | logger "Creating local cron script file" |
| 51 | touch /root/cron-script.sh |
| 52 | |
| 53 | # GCloud Init |
| 54 | logger "setting default project id for gcloud config" |
| 55 | gcloud config set project `jq -r .gcs_project_id /root/config.json` |
| 56 | |
| 57 | # Create Bucket Name |
| 58 | logger "creating log bucket in GCS" |
| 59 | export GSB=`jq -r .gcs_project_id /root/config.json`-logs-$RANDOMVALUE |
| 60 | |
| 61 | # Create Staging Bucket Name |
| 62 | logger "creating staging bucket for setup files" |
| 63 | export GSBSTAGING=`jq -r .gcs_project_id /root/config.json`-staging-$RANDOMVALUE |
| 64 | |
| 65 | |
| 66 | logger "provisioning logshare-cli command with cloudflare credentials" |
| 67 | echo -e "START=\`date +%s --date '-11 minutes'\`\nEND=\`date +%s --date '-10 minutes'\`\n\n/root/go/bin/logshare-cli --api-key=$APIKEY --api-email=$APIEMAIL --zone-name=$ZONENAME --count=-1 --google-storage-bucket=$GSB --google-project-id=`jq -r .gcs_project_id /root/config.json` --start-time=\$START --end-time=\$END --fields `cat /root/fields.txt` >> /root/logshare-cli.log 2>&1" > /root/cron-script.sh |
| 68 | |
| 69 | # Create two Buckets - One for the Logs and one for the Staging Files |
| 70 | logger "provisioning both gcs buckets" |
| 71 | gsutil mb -c regional -l us-central1 "gs://$GSB" |
| 72 | gsutil mb -c regional -l us-central1 "gs://$GSBSTAGING" |
| 73 | |
| 74 | # Configure the Cloud Function |
| 75 | logger "cloning the cloud function repo" |
| 76 | git clone https://github.com/cloudflare/GCS-To-Big-Query.git /root/GCS-To-Big-Query |
| 77 | |
| 78 | # Update the GCS config file with the project identifier |
| 79 | echo '{"DATASET": "cloudflare_logs_'$RANDOMVALUE'","TABLE": "cloudflare_els_'$RANDOMVALUE'"}' > /root/GCS-To-Big-Query/config.json |
| 80 | |
| 81 | logger "zipping up files for cloud function" |
| 82 | zip -j /root/archive.zip /root/GCS-To-Big-Query/* |
| 83 | |
| 84 | logger "copying setup files to staging bucket" |
| 85 | gsutil cp /root/archive.zip gs://$GSBSTAGING |
| 86 | |
| 87 | logger "deploying cloud function" |
| 88 | gcloud beta functions deploy cflogs_upload_bucket_$RANDOMVALUE --trigger-bucket=gs://$GSB --source=gs://$GSBSTAGING/archive.zip --stage-bucket=gs://$GSBSTAGING --entry-point=jsonLoad |
| 89 | |
| 90 | chmod +x /root/cron-script.sh |
| 91 | |
| 92 | logger "provisioning cronjob" |
| 93 | crontab -l > file; echo '* * * * * /root/cron-script.sh' >> file; crontab file |
| 94 | |
| 95 | else |
| 96 | logger "Second boot" |
| 97 | fi |