CodeCommitsIssuesPull requestsActionsInsightsSecurity
1f022e4db14b0e80269e85ae1bbcf5ed78790370

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

src/deploy.go

695lines · modecode

1package main
2
3import (
4 "bytes"
5 "compress/gzip"
6 "crypto/md5"
7 "encoding/base64"
8 "fmt"
9 "io"
10 "math/big"
11 "mime"
12 "net/url"
13 "os"
14 "os/exec"
15 "path/filepath"
16 "strings"
17 "sync"
18 "time"
19
20 "github.com/cenk/backoff"
21 "golang.org/x/net/html"
22
23 "log"
24
25 "github.com/wsxiaoys/terminal/color"
26 "github.com/zackbloom/goamz/s3"
27)
28
29const (
30 SCRIPT = iota
31 STYLE
32)
33
34const UPLOAD_WORKERS = 20
35
36var NO_GZIP = []string{
37 "mp4",
38 "webm",
39 "ogg",
40}
41
42func hashFile(path string) []byte {
43 hash := md5.New()
44 io.WriteString(hash, path)
45 io.WriteString(hash, "\n")
46
47 // TODO: Encode type?
48
49 ref := must(os.Open(path)).(*os.File)
50 defer ref.Close()
51
52 must(io.Copy(hash, ref))
53
54 return hash.Sum(nil)
55}
56
57func hashBytes(data []byte) []byte {
58 hash := md5.New()
59 must(io.Copy(hash, bytes.NewReader(data)))
60 return hash.Sum(nil)
61}
62
63func hashFiles(files []string) string {
64 hash := new(big.Int)
65 for _, file := range files {
66 val := new(big.Int)
67 val.SetBytes(hashFile(file))
68
69 hash = hash.Xor(hash, val)
70 }
71
72 return fmt.Sprintf("%x", hash)
73}
74
75func getRef() string {
76 gitPath := mustString(exec.LookPath("git"))
77
78 cmd := exec.Command(gitPath, "rev-parse", "--verify", "HEAD")
79
80 out := bytes.Buffer{}
81 cmd.Stdout = &out
82 panicIf(cmd.Run())
83
84 return string(out.Bytes())
85}
86
87func guessContentType(file string) string {
88 return mime.TypeByExtension(filepath.Ext(file))
89}
90
91func shouldCompress(file string) bool {
92 ext := filepath.Ext(file)
93 for _, e := range NO_GZIP {
94 if "."+e == ext {
95 return false
96 }
97 }
98
99 return true
100}
101
102type UploadFileRequest struct {
103 Bucket *s3.Bucket
104 Reader io.Reader
105 Path string
106 Dest string
107 IncludeHash bool
108 CacheSeconds int
109}
110
111func uploadFile(req UploadFileRequest) (remotePath string) {
112 buffer := bytes.NewBuffer([]byte{})
113
114 compress := shouldCompress(req.Path)
115
116 if compress {
117 writer := gzip.NewWriter(buffer)
118 must(io.Copy(writer, req.Reader))
119 writer.Close()
120 } else {
121 must(io.Copy(buffer, req.Reader))
122 }
123
124 data := buffer.Bytes()
125
126 hash := hashBytes(data)
127 hashPrefix := fmt.Sprintf("%x", hash)[:12]
128 s3Opts := s3.Options{
129 ContentMD5: base64.StdEncoding.EncodeToString(hash),
130 CacheControl: fmt.Sprintf("public, max-age=%d", req.CacheSeconds),
131 }
132
133 if compress {
134 s3Opts.ContentEncoding = "gzip"
135 }
136
137 dest := req.Path
138 if req.IncludeHash {
139 dest = hashPrefix + "_" + dest
140 }
141 dest = filepath.Join(req.Dest, dest)
142
143 log.Printf("Uploading to %s in %s (%s) [%d]\n", dest, req.Bucket.Name, hashPrefix, req.CacheSeconds)
144
145 op := func() error {
146 // We need to create a new reader each time, as we might be doing this more than once (if it fails)
147 return req.Bucket.PutReader(dest, bytes.NewReader(data), int64(len(data)), guessContentType(dest)+"; charset=utf-8", s3.PublicRead, s3Opts)
148 }
149
150 back := backoff.NewExponentialBackOff()
151 back.MaxElapsedTime = 30 * time.Second
152
153 err := backoff.RetryNotify(op, back, func(err error, next time.Duration) {
154 log.Println("Error uploading", err, "retrying in", next)
155 })
156 panicIf(err)
157
158 return dest
159}
160
161type FileRef struct {
162 LocalPath string
163 RemotePath string
164 UploadedPath string
165}
166
167type FileInst struct {
168 File *FileRef
169 InstPath string
170}
171
172func writeFiles(options Options, includeHash bool, files chan *FileRef) {
173 bucket := s3Session.Bucket(options.Bucket)
174
175 for file := range files {
176 handle := must(os.Open(file.LocalPath)).(*os.File)
177 defer handle.Close()
178
179 var ttl int
180 ttl = FOREVER
181 if !includeHash {
182 ttl = LIMITED
183 }
184
185 remote := file.RemotePath
186 if strings.HasPrefix(remote, "/") {
187 remote = remote[1:]
188 }
189 partialPath, err := filepath.Rel(options.Dest, remote)
190 if err != nil {
191 panic(err)
192 }
193
194 (*file).UploadedPath = uploadFile(UploadFileRequest{
195 Bucket: bucket,
196 Reader: handle,
197 Path: partialPath,
198 Dest: options.Dest,
199 IncludeHash: includeHash,
200 CacheSeconds: ttl,
201 })
202 }
203}
204
205func deployFiles(options Options, includeHash bool, files []*FileRef) {
206 ch := make(chan *FileRef)
207
208 wg := new(sync.WaitGroup)
209 for i := 0; i < UPLOAD_WORKERS; i++ {
210 wg.Add(1)
211 go func() {
212 writeFiles(options, includeHash, ch)
213 wg.Done()
214 }()
215 }
216
217 for _, file := range files {
218 if !includeHash && strings.HasSuffix(file.RemotePath, ".html") {
219 panic(fmt.Sprintf("Cowardly refusing to deploy an html file (%s) without versioning.", file.RemotePath))
220 }
221
222 ch <- file
223 }
224
225 close(ch)
226
227 wg.Wait()
228}
229
230func addFiles(form uint8, parent *html.Node, files []string) {
231 for _, file := range files {
232 node := html.Node{
233 Type: html.ElementNode,
234 }
235 switch form {
236 case SCRIPT:
237 node.Data = "script"
238 node.Attr = []html.Attribute{
239 html.Attribute{
240 Key: "src",
241 Val: file,
242 },
243 }
244
245 case STYLE:
246 node.Data = "link"
247 node.Attr = []html.Attribute{
248 html.Attribute{
249 Key: "rel",
250 Val: "stylesheet",
251 },
252 html.Attribute{
253 Key: "href",
254 Val: file,
255 },
256 }
257 default:
258 panic("Type not understood")
259 }
260
261 parent.AppendChild(&node)
262 }
263}
264
265func isLocal(href string) bool {
266 parsed := must(url.Parse(href)).(*url.URL)
267 return parsed.Host == ""
268}
269
270func formatHref(path string) string {
271 if !strings.HasPrefix(path, "/") {
272 path = "/" + path
273 }
274 return path
275}
276
277func renderHTML(options Options, file HTMLFile) string {
278 handle := must(os.Open(file.File.LocalPath)).(*os.File)
279 defer handle.Close()
280
281 doc := must(html.Parse(handle)).(*html.Node)
282
283 var f func(*html.Node)
284 f = func(n *html.Node) {
285 for c := n.FirstChild; c != nil; c = c.NextSibling {
286 f(c)
287 }
288
289 if n.Type == html.ElementNode {
290 switch n.Data {
291 case "script":
292 for i, a := range n.Attr {
293 if a.Key == "src" {
294 for _, dep := range file.Deps {
295 if dep.InstPath == a.Val {
296 n.Attr[i].Val = formatHref(dep.File.UploadedPath)
297 break
298 }
299 }
300 }
301 }
302 case "link":
303 stylesheet := false
304 for _, a := range n.Attr {
305 if a.Key == "rel" {
306 stylesheet = a.Val == "stylesheet"
307 break
308 }
309 }
310 if !stylesheet {
311 return
312 }
313
314 for i, a := range n.Attr {
315 if a.Key == "href" {
316 for _, dep := range file.Deps {
317 if dep.InstPath == a.Val {
318 n.Attr[i].Val = formatHref(dep.File.UploadedPath)
319 break
320 }
321 }
322 }
323 }
324 }
325 }
326 }
327 f(doc)
328
329 buf := bytes.NewBuffer([]byte{})
330 panicIf(html.Render(buf, doc))
331
332 return buf.String()
333}
334
335func parseHTML(options Options, path string) (files []string, base string) {
336 files = make([]string, 0)
337
338 handle := must(os.Open(path)).(*os.File)
339 defer handle.Close()
340
341 doc := must(html.Parse(handle)).(*html.Node)
342
343 var f func(*html.Node)
344 f = func(n *html.Node) {
345 for c := n.FirstChild; c != nil; c = c.NextSibling {
346 f(c)
347 }
348
349 if n.Type == html.ElementNode {
350 switch n.Data {
351 case "base":
352 for _, a := range n.Attr {
353 if a.Key == "href" {
354 base = a.Val
355 }
356 }
357 case "script":
358 for _, a := range n.Attr {
359 if a.Key == "src" {
360 if isLocal(a.Val) {
361 files = append(files, a.Val)
362 }
363 }
364 }
365 case "link":
366 local := false
367 stylesheet := false
368 href := ""
369 for _, a := range n.Attr {
370 switch a.Key {
371 case "href":
372 local = isLocal(a.Val)
373 href = a.Val
374 case "rel":
375 stylesheet = a.Val == "stylesheet"
376 }
377 }
378 if local && stylesheet {
379 files = append(files, href)
380 }
381 }
382 }
383 }
384 f(doc)
385
386 return
387}
388
389func deployHTML(options Options, id string, file HTMLFile) {
390 data := renderHTML(options, file)
391
392 internalPath, err := filepath.Rel(options.Root, file.File.LocalPath)
393 if err != nil {
394 panic(err)
395 }
396
397 permPath := joinPath(options.Dest, id, internalPath)
398 curPath := joinPath(options.Dest, internalPath)
399
400 bucket := s3Session.Bucket(options.Bucket)
401 uploadFile(UploadFileRequest{
402 Bucket: bucket,
403 Reader: strings.NewReader(data),
404 Path: permPath,
405 IncludeHash: false,
406 CacheSeconds: FOREVER,
407 })
408
409 log.Println("Copying", permPath, "to", curPath)
410 copyFile(bucket, permPath, curPath, "text/html; charset=utf-8", LIMITED)
411}
412
413func expandFiles(root string, glob string) []string {
414 out := make([]string, 0)
415 cases := strings.Split(glob, ",")
416
417 for _, pattern := range cases {
418 if strings.HasPrefix(pattern, "-/") {
419 pattern = pattern[2:]
420 } else {
421 pattern = joinPath(root, pattern)
422 }
423
424 list := must(filepath.Glob(pattern)).([]string)
425
426 for _, file := range list {
427 info := must(os.Stat(file)).(os.FileInfo)
428
429 if info.IsDir() {
430 filepath.Walk(file, func(path string, info os.FileInfo, err error) error {
431 panicIf(err)
432
433 if !info.IsDir() {
434 out = append(out, path)
435 }
436
437 return nil
438 })
439 } else {
440 out = append(out, file)
441 }
442 }
443 }
444 return out
445}
446
447func listFiles(options Options) []*FileRef {
448 filePaths := expandFiles(options.Root, options.Files)
449
450 files := make([]*FileRef, len(filePaths))
451 for i, path := range filePaths {
452 remotePath := joinPath(options.Dest, mustString(filepath.Rel(options.Root, path)))
453
454 for strings.HasPrefix(remotePath, "../") {
455 remotePath = remotePath[3:]
456 }
457
458 files[i] = &FileRef{
459 LocalPath: path,
460 RemotePath: remotePath,
461 }
462 }
463
464 return files
465}
466
467func ignoreFiles(full []*FileRef, rem []*FileRef) []*FileRef {
468 out := make([]*FileRef, 0, len(full))
469
470 for _, file := range full {
471 ignore := false
472 path := filepath.Clean(file.LocalPath)
473
474 for _, remFile := range rem {
475 if filepath.Clean(remFile.LocalPath) == path {
476 ignore = true
477 break
478 }
479 }
480
481 if !ignore {
482 out = append(out, file)
483 }
484 }
485
486 return out
487}
488
489func extractFileList(options Options, pattern string) (files []string) {
490 files = make([]string, 0)
491
492 parts := strings.Split(pattern, ",")
493
494 for _, part := range parts {
495 matches, err := filepath.Glob(joinPath(options.Root, part))
496 if err != nil {
497 panic(err)
498 }
499 if matches == nil {
500 panic(fmt.Sprintf("Pattern %s did not match any files", part))
501 }
502
503 files = append(files, matches...)
504 }
505
506 return files
507}
508
509func filesWithExtension(files []*FileRef, ext string) (outFiles []*FileRef) {
510 outFiles = make([]*FileRef, 0)
511 for _, file := range files {
512 if filepath.Ext(file.LocalPath) == ext {
513 outFiles = append(outFiles, file)
514 }
515 }
516
517 return
518}
519
520type HTMLFile struct {
521 File FileRef
522 Deps []FileInst
523 Base string
524}
525
526func (f HTMLFile) GetLocalPath() string {
527 return f.File.LocalPath
528}
529
530func Deploy(options Options) {
531 if s3Session == nil {
532 s3Session = openS3(options.AWSKey, options.AWSSecret, options.AWSRegion)
533 }
534
535 files := listFiles(options)
536
537 htmlFileRefs := filesWithExtension(files, ".html")
538 var htmlFiles []HTMLFile
539 var id string
540
541 if len(htmlFileRefs) == 0 {
542 log.Println("No HTML files found")
543 } else {
544 inclFiles := make(map[string]*FileRef)
545 htmlFiles = make([]HTMLFile, len(htmlFileRefs))
546 for i, file := range htmlFileRefs {
547 dir := filepath.Dir(file.LocalPath)
548
549 rel, err := filepath.Rel(options.Root, dir)
550 if err != nil {
551 panic(err)
552 }
553
554 paths, base := parseHTML(options, file.LocalPath)
555
556 if strings.HasPrefix(strings.ToLower(base), "http") || strings.HasPrefix(base, "//") {
557 panic("Absolute base tags are not supported")
558 }
559
560 if strings.HasSuffix(base, "/") {
561 base = base[:len(base)-1]
562 }
563
564 htmlFiles[i] = HTMLFile{
565 File: *file,
566 Deps: make([]FileInst, len(paths)),
567 Base: base,
568 }
569
570 var dest string
571 if strings.HasPrefix(base, "/") && strings.HasPrefix(base, "/"+options.Dest) {
572 dest = base
573 } else {
574 dest = joinPath(options.Dest, base)
575 }
576
577 var root string
578 if strings.HasPrefix(base, "/") && strings.HasSuffix(options.Root, base) {
579 root = options.Root
580 } else {
581 root = joinPath(options.Root, base)
582 }
583
584 for j, path := range paths {
585 var local, remote string
586 if strings.HasPrefix(path, "/") {
587 local = joinPath(options.Root, path)
588 remote = joinPath(options.Dest, path)
589 } else {
590 if strings.HasPrefix(base, "/") {
591 local = joinPath(root, path)
592 remote = joinPath(dest, path)
593 } else {
594 local = joinPath(options.Root, rel, base, path)
595 remote = joinPath(options.Dest, rel, base, path)
596 }
597 }
598
599 for strings.HasPrefix(remote, "../") {
600 remote = remote[3:]
601 }
602
603 ref, ok := inclFiles[local]
604 if !ok {
605 ref = &FileRef{
606 LocalPath: local,
607 RemotePath: remote,
608
609 // Filled in after the deploy:
610 UploadedPath: "",
611 }
612
613 inclFiles[local] = ref
614 }
615
616 use := FileInst{
617 File: ref,
618 InstPath: path,
619 }
620
621 htmlFiles[i].Deps[j] = use
622 }
623 }
624
625 inclFileList := make([]*FileRef, len(inclFiles))
626 i := 0
627 for _, ref := range inclFiles {
628 inclFileList[i] = ref
629 i++
630 }
631
632 hashPaths := make([]string, 0)
633 for _, item := range inclFileList {
634 hashPaths = append(hashPaths, item.LocalPath)
635 }
636 for _, item := range htmlFiles {
637 hashPaths = append(hashPaths, item.File.LocalPath)
638 }
639
640 hash := hashFiles(hashPaths)
641 id = hash[:12]
642
643 deployFiles(options, true, inclFileList)
644 }
645
646 deployFiles(options, false, ignoreFiles(files, htmlFileRefs))
647
648 if len(htmlFileRefs) != 0 {
649 // Ensure that the new files exist in s3
650 // Time based on "Eventual Consistency: How soon is eventual?"
651 time.Sleep(1500 * time.Millisecond)
652
653 wg := sync.WaitGroup{}
654 for _, file := range htmlFiles {
655 wg.Add(1)
656
657 go func(file HTMLFile) {
658 defer wg.Done()
659 deployHTML(options, id, file)
660 }(file)
661 }
662
663 wg.Wait()
664 }
665
666 visId := id
667 if id == "" {
668 visId = "0 HTML Files"
669 }
670
671 color.Printf(`
672+------------------------------------+
673| @{g}Deploy Successful!@{|} |
674| |
675| Deploy ID: @{?}%s@{|} |
676+------------------------------------+
677`, visId)
678
679}
680
681func deployCmd() {
682 options, _ := parseOptions()
683 loadConfigFile(&options)
684 addAWSConfig(&options)
685
686 if options.Bucket == "" {
687 panic("You must specify a bucket")
688 }
689
690 if options.AWSKey == "" || options.AWSSecret == "" {
691 panic("You must specify your AWS credentials")
692 }
693
694 Deploy(options)
695}