diff --git a/cmd/syncthing/gui.go b/cmd/syncthing/gui.go index b276fe79..ca431e6b 100644 --- a/cmd/syncthing/gui.go +++ b/cmd/syncthing/gui.go @@ -969,7 +969,9 @@ func (s *apiService) getRandomString(w http.ResponseWriter, r *http.Request) { func (s *apiService) getDBIgnores(w http.ResponseWriter, r *http.Request) { qs := r.URL.Query() - ignores, patterns, err := s.model.GetIgnores(qs.Get("folder")) + folder := qs.Get("folder") + + ignores, patterns, err := s.model.GetIgnores(folder) if err != nil { http.Error(w, err.Error(), 500) return diff --git a/gui/default/assets/lang/lang-en.json b/gui/default/assets/lang/lang-en.json index 26db4838..24d54112 100644 --- a/gui/default/assets/lang/lang-en.json +++ b/gui/default/assets/lang/lang-en.json @@ -43,6 +43,7 @@ "Copied from original": "Copied from original", "Copyright © 2014-2016 the following Contributors:": "Copyright © 2014-2016 the following Contributors:", "Copyright © 2014-2017 the following Contributors:": "Copyright © 2014-2017 the following Contributors:", + "Creating ignore patterns, overwriting an existing file at {%path%}.": "Creating ignore patterns, overwriting an existing file at {{path}}.", "Danger!": "Danger!", "Deleted": "Deleted", "Device": "Device", @@ -63,6 +64,7 @@ "Edit Device": "Edit Device", "Edit Folder": "Edit Folder", "Editing": "Editing", + "Editing {%path%}.": "Editing {{path}}.", "Enable NAT traversal": "Enable NAT traversal", "Enable Relaying": "Enable Relaying", "Enter comma separated (\"tcp://ip:port\", \"tcp://host:port\") addresses or \"dynamic\" to perform automatic discovery of the address.": "Enter comma separated (\"tcp://ip:port\", \"tcp://host:port\") addresses or \"dynamic\" to perform automatic discovery of the address.", diff --git a/gui/default/syncthing/core/syncthingController.js b/gui/default/syncthing/core/syncthingController.js index f1717681..a418e2d0 100755 --- a/gui/default/syncthing/core/syncthingController.js +++ b/gui/default/syncthing/core/syncthingController.js @@ -58,6 +58,24 @@ angular.module('syncthing.core') $scope.metricRates = (window.localStorage["metricRates"] == "true"); } catch (exception) { } + $scope.folderDefaults = { + selectedDevices: {}, + type: "readwrite", + rescanIntervalS: 60, + minDiskFreePct: 1, + maxConflicts: 10, + fsync: true, + order: "random", + fileVersioningSelector: "none", + trashcanClean: 0, + simpleKeep: 5, + staggeredMaxAge: 365, + staggeredCleanInterval: 3600, + staggeredVersionsPath: "", + externalCommand: "", + autoNormalize: true + }; + $scope.localStateTotal = { bytes: 0, files: 0 @@ -1393,24 +1411,9 @@ angular.module('syncthing.core') }; $scope.addFolder = function () { - $scope.currentFolder = { - selectedDevices: {}, - type: "readwrite", - rescanIntervalS: 60, - minDiskFreePct: 1, - maxConflicts: 10, - fsync: true, - order: "random", - fileVersioningSelector: "none", - trashcanClean: 0, - simpleKeep: 5, - staggeredMaxAge: 365, - staggeredCleanInterval: 3600, - staggeredVersionsPath: "", - externalCommand: "", - autoNormalize: true - }; + $scope.currentFolder = angular.copy($scope.folderDefaults); $scope.editingExisting = false; + $('#editIgnores textarea').val(""); $scope.folderEditor.$setPristine(); $http.get(urlbase + '/svc/random/string?length=10').success(function (data) { $scope.currentFolder.id = (data.random.substr(0, 5) + '-' + data.random.substr(5, 5)).toLowerCase(); @@ -1420,26 +1423,11 @@ angular.module('syncthing.core') $scope.addFolderAndShare = function (folder, folderLabel, device) { $scope.dismissFolderRejection(folder, device); - $scope.currentFolder = { - id: folder, - label: folderLabel, - selectedDevices: {}, - rescanIntervalS: 60, - minDiskFreePct: 1, - maxConflicts: 10, - fsync: true, - order: "random", - fileVersioningSelector: "none", - trashcanClean: 0, - simpleKeep: 5, - staggeredMaxAge: 365, - staggeredCleanInterval: 3600, - staggeredVersionsPath: "", - externalCommand: "", - autoNormalize: true, - viewFlags: { - importFromOtherDevice: true - } + $scope.currentFolder = angular.copy($scope.folderDefaults); + $scope.currentFolder.id = folder; + $scope.currentFolder.label = folderLabel; + $scope.currentFolder.viewFlags = { + importFromOtherDevice: true }; $scope.currentFolder.selectedDevices[device] = true; @@ -1516,10 +1504,20 @@ angular.module('syncthing.core') delete folderCfg.versioning; } + var ignores = $('#editIgnores textarea').val().trim(); + if (!$scope.editingExisting && ignores) { + folderCfg.paused = true; + }; + $scope.folders[folderCfg.id] = folderCfg; $scope.config.folders = folderList($scope.folders); $scope.saveConfig(); + + if (!$scope.editingExisting && ignores) { + $scope.saveIgnores(); + $scope.setFolderPause(folderCfg.id, false); + }; }; $scope.dismissFolderRejection = function (folder, device) { @@ -1593,11 +1591,21 @@ angular.module('syncthing.core') }); }; - $scope.saveIgnores = function () { - if (!$scope.editingExisting) { + $scope.editIgnoresOnAddingFolder = function () { + if ($scope.editingExisting) { return; } + if ($scope.currentFolder.path.endsWith($scope.system.pathSeparator)) { + $scope.currentFolder.path = $scope.currentFolder.path.slice(0, -1); + }; + $('#editIgnores').modal().one('shown.bs.modal', function () { + textArea.focus(); + }); + }; + + + $scope.saveIgnores = function () { $http.post(urlbase + '/db/ignores?folder=' + encodeURIComponent($scope.currentFolder.id), { ignore: $('#editIgnores textarea').val().split('\n') }); diff --git a/gui/default/syncthing/folder/editFolderModalView.html b/gui/default/syncthing/folder/editFolderModalView.html index 2eb1faee..359f90e1 100644 --- a/gui/default/syncthing/folder/editFolderModalView.html +++ b/gui/default/syncthing/folder/editFolderModalView.html @@ -184,7 +184,7 @@ - - \ No newline at end of file + diff --git a/lib/ignore/ignore.go b/lib/ignore/ignore.go index 81bc1a88..20338ea9 100644 --- a/lib/ignore/ignore.go +++ b/lib/ignore/ignore.go @@ -19,6 +19,7 @@ import ( "time" "github.com/gobwas/glob" + "github.com/syncthing/syncthing/lib/osutil" "github.com/syncthing/syncthing/lib/sync" ) @@ -64,6 +65,7 @@ func (r Result) IsCaseFolded() bool { } type Matcher struct { + lines []string patterns []Pattern withCache bool matches *cache @@ -120,7 +122,7 @@ func (m *Matcher) Parse(r io.Reader, file string) error { } func (m *Matcher) parseLocked(r io.Reader, file string) error { - patterns, err := parseIgnoreFile(r, file, m.modtimes) + lines, patterns, err := parseIgnoreFile(r, file, m.modtimes) // Error is saved and returned at the end. We process the patterns // (possibly blank) anyway. @@ -131,6 +133,7 @@ func (m *Matcher) parseLocked(r io.Reader, file string) error { } m.curHash = newHash + m.lines = lines m.patterns = patterns if m.withCache { m.matches = newCache(patterns) @@ -206,6 +209,13 @@ func (m *Matcher) Match(file string) (result Result) { return resultNotMatched } +// Lines return a list of the unprocessed lines in .stignore at last load +func (m *Matcher) Lines() []string { + m.mut.Lock() + defer m.mut.Unlock() + return m.lines +} + // Patterns return a list of the loaded patterns, as they've been parsed func (m *Matcher) Patterns() []string { if m == nil { @@ -274,27 +284,28 @@ func hashPatterns(patterns []Pattern) string { return fmt.Sprintf("%x", h.Sum(nil)) } -func loadIgnoreFile(file string, modtimes map[string]time.Time) ([]Pattern, error) { +func loadIgnoreFile(file string, modtimes map[string]time.Time) ([]string, []Pattern, error) { if _, ok := modtimes[file]; ok { - return nil, fmt.Errorf("Multiple include of ignore file %q", file) + return nil, nil, fmt.Errorf("multiple include of ignore file %q", file) } fd, err := os.Open(file) if err != nil { - return nil, err + return nil, nil, err } defer fd.Close() info, err := fd.Stat() if err != nil { - return nil, err + return nil, nil, err } modtimes[file] = info.ModTime() return parseIgnoreFile(fd, file, modtimes) } -func parseIgnoreFile(fd io.Reader, currentFile string, modtimes map[string]time.Time) ([]Pattern, error) { +func parseIgnoreFile(fd io.Reader, currentFile string, modtimes map[string]time.Time) ([]string, []Pattern, error) { + var lines []string var patterns []Pattern defaultResult := resultInclude @@ -360,11 +371,12 @@ func parseIgnoreFile(fd io.Reader, currentFile string, modtimes map[string]time. } else if strings.HasPrefix(line, "#include ") { includeRel := line[len("#include "):] includeFile := filepath.Join(filepath.Dir(currentFile), includeRel) - includes, err := loadIgnoreFile(includeFile, modtimes) + includeLines, includePatterns, err := loadIgnoreFile(includeFile, modtimes) if err != nil { return fmt.Errorf("include of %q: %v", includeRel, err) } - patterns = append(patterns, includes...) + lines = append(lines, includeLines...) + patterns = append(patterns, includePatterns...) } else { // Path name or pattern, add it so it matches files both in // current directory and subdirs. @@ -389,6 +401,7 @@ func parseIgnoreFile(fd io.Reader, currentFile string, modtimes map[string]time. var err error for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) + lines = append(lines, line) switch { case line == "": continue @@ -411,11 +424,11 @@ func parseIgnoreFile(fd io.Reader, currentFile string, modtimes map[string]time. } } if err != nil { - return nil, err + return nil, nil, err } } - return patterns, nil + return lines, patterns, nil } // IsInternal returns true if the file, as a path relative to the folder @@ -434,3 +447,22 @@ func IsInternal(file string) bool { } return false } + +// WriteIgnores is a convenience function to avoid code duplication +func WriteIgnores(path string, content []string) error { + fd, err := osutil.CreateAtomic(path) + if err != nil { + return err + } + + for _, line := range content { + fmt.Fprintln(fd, line) + } + + if err := fd.Close(); err != nil { + return err + } + osutil.HideFile(path) + + return nil +} diff --git a/lib/model/folder.go b/lib/model/folder.go index 927519f4..49e20b87 100644 --- a/lib/model/folder.go +++ b/lib/model/folder.go @@ -10,9 +10,11 @@ import "time" type folder struct { stateTracker - scan folderScanner - model *Model - stop chan struct{} + + scan folderScanner + model *Model + stop chan struct{} + initialScanCompleted chan struct{} } func (f *folder) IndexUpdated() { @@ -23,6 +25,7 @@ func (f *folder) DelayScan(next time.Duration) { } func (f *folder) Scan(subdirs []string) error { + <-f.initialScanCompleted return f.scan.Scan(subdirs) } func (f *folder) Stop() { diff --git a/lib/model/model.go b/lib/model/model.go index e38c0479..f09401f9 100644 --- a/lib/model/model.go +++ b/lib/model/model.go @@ -7,7 +7,6 @@ package model import ( - "bufio" "crypto/tls" "encoding/json" "errors" @@ -1252,66 +1251,51 @@ func (m *Model) ConnectedTo(deviceID protocol.DeviceID) bool { } func (m *Model) GetIgnores(folder string) ([]string, []string, error) { - var lines []string - m.fmut.RLock() cfg, ok := m.folderCfgs[folder] m.fmut.RUnlock() - if !ok { - return lines, nil, fmt.Errorf("Folder %s does not exist", folder) - } - - if !cfg.HasMarker() { - return lines, nil, fmt.Errorf("Folder %s stopped", folder) - } - - fd, err := os.Open(filepath.Join(cfg.Path(), ".stignore")) - if err != nil { - if os.IsNotExist(err) { - return lines, nil, nil + if ok { + if !cfg.HasMarker() { + return nil, nil, fmt.Errorf("Folder %s stopped", folder) } - l.Warnln("Loading .stignore:", err) - return lines, nil, err - } - defer fd.Close() - scanner := bufio.NewScanner(fd) - for scanner.Scan() { - lines = append(lines, strings.TrimSpace(scanner.Text())) + m.fmut.RLock() + ignores := m.folderIgnores[folder] + m.fmut.RUnlock() + + return ignores.Lines(), ignores.Patterns(), nil } - m.fmut.RLock() - patterns := m.folderIgnores[folder].Patterns() - m.fmut.RUnlock() + if cfg, ok := m.cfg.Folders()[folder]; ok { + matcher := ignore.New(false) + path := filepath.Join(cfg.Path(), ".stignore") + if err := matcher.Load(path); err != nil { + return nil, nil, err + } + return matcher.Lines(), matcher.Patterns(), nil + } - return lines, patterns, nil + return nil, nil, fmt.Errorf("Folder %s does not exist", folder) } func (m *Model) SetIgnores(folder string, content []string) error { - cfg, ok := m.folderCfgs[folder] + cfg, ok := m.cfg.Folders()[folder] if !ok { return fmt.Errorf("Folder %s does not exist", folder) } - path := filepath.Join(cfg.Path(), ".stignore") - - fd, err := osutil.CreateAtomic(path) - if err != nil { + if err := ignore.WriteIgnores(filepath.Join(cfg.Path(), ".stignore"), content); err != nil { l.Warnln("Saving .stignore:", err) return err } - for _, line := range content { - fmt.Fprintln(fd, line) + m.fmut.RLock() + runner, ok := m.folderRunners[folder] + m.fmut.RUnlock() + if ok { + return runner.Scan(nil) } - - if err := fd.Close(); err != nil { - l.Warnln("Saving .stignore:", err) - return err - } - osutil.HideFile(path) - - return m.ScanFolder(folder) + return nil } // OnHello is called when an device connects to us. @@ -2395,9 +2379,13 @@ func (m *Model) CommitConfiguration(from, to config.Configuration) bool { for folderID, cfg := range toFolders { if _, ok := fromFolders[folderID]; !ok { // A folder was added. - l.Debugln(m, "adding folder", folderID) - m.AddFolder(cfg) - m.StartFolder(folderID) + if cfg.Paused { + l.Infoln(m, "Paused folder", cfg.Description()) + } else { + l.Infoln(m, "Adding folder", cfg.Description()) + m.AddFolder(cfg) + m.StartFolder(folderID) + } } } diff --git a/lib/model/model_test.go b/lib/model/model_test.go index fff14880..6dd86109 100644 --- a/lib/model/model_test.go +++ b/lib/model/model_test.go @@ -927,7 +927,7 @@ func TestIntroducer(t *testing.T) { } } -func TestIgnores(t *testing.T) { +func changeIgnores(t *testing.T, m *Model, expected []string) { arrEqual := func(a, b []string) bool { if len(a) != len(b) { return false @@ -941,22 +941,6 @@ func TestIgnores(t *testing.T) { return true } - // Assure a clean start state - ioutil.WriteFile("testdata/.stfolder", nil, 0644) - ioutil.WriteFile("testdata/.stignore", []byte(".*\nquux\n"), 0644) - - db := db.OpenMemory() - m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil) - m.AddFolder(defaultFolderConfig) - m.StartFolder("default") - m.ServeBackground() - defer m.Stop() - - expected := []string{ - ".*", - "quux", - } - ignores, _, err := m.GetIgnores("default") if err != nil { t.Error(err) @@ -999,8 +983,34 @@ func TestIgnores(t *testing.T) { if !arrEqual(ignores, expected) { t.Errorf("Incorrect ignores: %v != %v", ignores, expected) } +} - _, _, err = m.GetIgnores("doesnotexist") +func TestIgnores(t *testing.T) { + // Assure a clean start state + ioutil.WriteFile("testdata/.stfolder", nil, 0644) + ioutil.WriteFile("testdata/.stignore", []byte(".*\nquux\n"), 0644) + + db := db.OpenMemory() + m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil) + m.ServeBackground() + defer m.Stop() + + // m.cfg.SetFolder is not usable as it is non-blocking, and there is no + // way to know when the folder is actually added. + m.AddFolder(defaultFolderConfig) + m.StartFolder("default") + + // Make sure the initial scan has finished (ScanFolders is blocking) + m.ScanFolders() + + expected := []string{ + ".*", + "quux", + } + + changeIgnores(t, m, expected) + + _, _, err := m.GetIgnores("doesnotexist") if err == nil { t.Error("No error") } @@ -1016,6 +1026,16 @@ func TestIgnores(t *testing.T) { if err == nil { t.Error("No error") } + + // Repeat tests with paused folder + pausedDefaultFolderConfig := defaultFolderConfig + pausedDefaultFolderConfig.Paused = true + + m.RestartFolder(pausedDefaultFolderConfig) + // Here folder initialization is not an issue as a paused folder isn't + // added to the model and thus there is no initial scan happening. + + changeIgnores(t, m, expected) } func TestROScanRecovery(t *testing.T) { @@ -1763,13 +1783,8 @@ func TestIssue3028(t *testing.T) { m.StartFolder("default") m.ServeBackground() - // Ugly hack for testing: reach into the model for the SendReceiveFolder and wait - // for it to complete the initial scan. The risk is that it otherwise - // runs during our modifications and screws up the test. - m.fmut.RLock() - folder := m.folderRunners["default"].(*sendReceiveFolder) - m.fmut.RUnlock() - <-folder.initialScanCompleted + // Make sure the initial scan has finished (ScanFolders is blocking) + m.ScanFolders() // Get a count of how many files are there now diff --git a/lib/model/rofolder.go b/lib/model/rofolder.go index 9f50ece3..2cbb9938 100644 --- a/lib/model/rofolder.go +++ b/lib/model/rofolder.go @@ -26,10 +26,11 @@ type sendOnlyFolder struct { func newSendOnlyFolder(model *Model, cfg config.FolderConfiguration, _ versioner.Versioner, _ *fs.MtimeFS) service { return &sendOnlyFolder{ folder: folder{ - stateTracker: newStateTracker(cfg.ID), - scan: newFolderScanner(cfg), - stop: make(chan struct{}), - model: model, + stateTracker: newStateTracker(cfg.ID), + scan: newFolderScanner(cfg), + stop: make(chan struct{}), + model: model, + initialScanCompleted: make(chan struct{}), }, FolderConfiguration: cfg, } @@ -43,7 +44,6 @@ func (f *sendOnlyFolder) Serve() { f.scan.timer.Stop() }() - initialScanCompleted := false for { select { case <-f.stop: @@ -68,9 +68,11 @@ func (f *sendOnlyFolder) Serve() { continue } - if !initialScanCompleted { + select { + case <-f.initialScanCompleted: + default: l.Infoln("Completed initial scan (ro) of", f.Description()) - initialScanCompleted = true + close(f.initialScanCompleted) } if f.scan.HasNoInterval() { diff --git a/lib/model/rwfolder.go b/lib/model/rwfolder.go index b6c29a5c..6fb073dc 100644 --- a/lib/model/rwfolder.go +++ b/lib/model/rwfolder.go @@ -96,17 +96,16 @@ type sendReceiveFolder struct { errors map[string]string // path -> error string errorsMut sync.Mutex - - initialScanCompleted chan (struct{}) // exposed for testing } func newSendReceiveFolder(model *Model, cfg config.FolderConfiguration, ver versioner.Versioner, mtimeFS *fs.MtimeFS) service { f := &sendReceiveFolder{ folder: folder{ - stateTracker: newStateTracker(cfg.ID), - scan: newFolderScanner(cfg), - stop: make(chan struct{}), - model: model, + stateTracker: newStateTracker(cfg.ID), + scan: newFolderScanner(cfg), + stop: make(chan struct{}), + model: model, + initialScanCompleted: make(chan struct{}), }, FolderConfiguration: cfg, @@ -119,8 +118,6 @@ func newSendReceiveFolder(model *Model, cfg config.FolderConfiguration, ver vers remoteIndex: make(chan struct{}, 1), // This needs to be 1-buffered so that we queue a notification if we're busy doing a pull when it comes. errorsMut: sync.NewMutex(), - - initialScanCompleted: make(chan struct{}), } f.configureCopiersAndPullers() @@ -1063,7 +1060,7 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- c // sweep is complete. As we do retries, we'll queue the scan // for this file up to ten times, but the last nine of those // scans will be cheap... - go f.scan.Scan([]string{file.Name}) + go f.Scan([]string{file.Name}) return } } diff --git a/lib/model/rwfolder_test.go b/lib/model/rwfolder_test.go index a3ea49c6..13a2fa7c 100644 --- a/lib/model/rwfolder_test.go +++ b/lib/model/rwfolder_test.go @@ -77,11 +77,12 @@ func setUpModel(file protocol.FileInfo) *Model { return model } -func setUpSendReceiveFolder(model *Model) sendReceiveFolder { - return sendReceiveFolder{ +func setUpSendReceiveFolder(model *Model) *sendReceiveFolder { + f := &sendReceiveFolder{ folder: folder{ - stateTracker: newStateTracker("default"), - model: model, + stateTracker: newStateTracker("default"), + model: model, + initialScanCompleted: make(chan struct{}), }, mtimeFS: fs.NewMtimeFS(fs.DefaultFilesystem, db.NewNamespacedKV(model.db, "mtime")), @@ -90,6 +91,11 @@ func setUpSendReceiveFolder(model *Model) sendReceiveFolder { errors: make(map[string]string), errorsMut: sync.NewMutex(), } + + // Folders are never actually started, so no initial scan will be done + close(f.initialScanCompleted) + + return f } // Layout of the files: (indexes from the above array)