Merge branch 'issue-23' into 'master'

Set macos maximum open file limit to 4096

Closes #23

See merge request !37

bob/devtools/log.py

@@ -229,18 +229,3 @@ def verbosity_option(**kwargs):
             "(e.g. '-vvv' for debug).",
             callback=callback, **kwargs)(f)
     return custom_verbosity_option
-
-
-def open_files():
-    '''Returns the number of open file descriptors for current process
-
-    .. warning: will only work on UNIX-like os-es.
-
-    '''
-
-    import os
-    import subprocess
-
-    pid = os.getpid()
-    procs = subprocess.check_output(['lsof', '-w', '-p', str(pid)])
-    return [k.split()[-1].decode('ascii') for k in procs.split(b'\n') if k][1:]

bob/devtools/scripts/ci.py

@@ -16,7 +16,7 @@ from ..constants import SERVER, CONDA_BUILD_CONFIG, CONDA_RECIPE_APPEND, \
     WEBDAV_PATHS, BASE_CONDARC
 from ..deploy import deploy_conda_package, deploy_documentation
 
-from ..log import verbosity_option, get_logger, echo_normal, open_files
+from ..log import verbosity_option, get_logger, echo_normal
 logger = get_logger(__name__)
 
 
@@ -509,10 +509,6 @@ def nightlies(ctx, order, dry_run):
     private = urlopen('https://gitlab.idiap.ch/%s' % package).getcode() != 200
     stable = 'STABLE' in os.environ
 
-    current_open_files = open_files()
-    logger.warn('Number of open files before build: %d',
-      len(current_open_files))
-
     ctx.invoke(build,
         recipe_dir=[os.path.join(clone_to, 'conda')],
         python=os.environ['PYTHON_VERSION'],  #python version
@@ -528,11 +524,6 @@ def nightlies(ctx, order, dry_run):
         ci=True,
         )
 
-    after_open_files = open_files()
-    logger.warn('Number of open files after build: %d', len(after_open_files))
-    logger.warn('New files opened: %s',
-        ', '.join(list(set(after_open_files) - set(current_open_files))))
-
     is_master = os.environ['CI_COMMIT_REF_NAME'] == 'master'
 
     # re-deploys a new conda package if it was rebuilt and it is the master

doc/macos-ci-install/install-homebrew.sh

@@ -14,7 +14,7 @@ fi
 ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" </dev/null
 brew=/usr/local/bin/brew
 
-${brew} install curl git coreutils bash-completion highlight neovim tmux htop python@3
+${brew} install curl git coreutils bash bash-completion highlight neovim tmux htop python@3
 ${brew} link --force curl #keg-only recipe
 ${brew} cask install mactex
 

doc/macos.rst

@@ -73,7 +73,46 @@ Building the reference setup
    execute pieces of the script by hand if something fails.  In that case,
    please investigate why it fails and properly fix the scripts so the next
    install runs more smoothly.
-6. Enter as gitlab user and install/configure the `gitlab runner`_:
+6. Check the maximum number of files that can be opened on a shell session
+   with the command ``launchctl limit maxfiles``.  If smaller than 4096, set
+   the maximum number of open files to 4096 by creating the file
+   ``/Library/LaunchDaemons/limit.maxfiles.plist`` with the following
+   contents::
+
+     <?xml version="1.0" encoding="UTF-8"?>
+     <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+       <plist version="1.0">
+         <dict>
+           <key>Label</key>
+             <string>limit.maxfiles</string>
+           <key>ProgramArguments</key>
+             <array>
+               <string>launchctl</string>
+               <string>limit</string>
+               <string>maxfiles</string>
+               <string>4096</string>
+               <string>unlimited</string>
+             </array>
+           <key>RunAtLoad</key>
+             <true/>
+           <key>ServiceIPC</key>
+             <false/>
+         </dict>
+       </plist>
+
+   At this occasion, verify if the kernel limits are not lower than this value
+   using::
+
+     $ sysctl kern.maxfilesperproc
+     10240  #example output
+     $ sysctl kern.maxfiles
+     12288  #example output
+
+   If that is the case (i.e., the values are lower than 4096), set those values
+   so they are slightly higher than that new limit with ``sudo sysctl -w
+   kern.maxfilesperproc=10240`` and ``sudo sysctl -w kern.maxfiles=12288``
+   respectively, for example.
+7. Enter as gitlab user and install/configure the `gitlab runner`_:
 
    Configure the runner for `shell executor`_, with local caching.  As
    ``gitlab`` user, execute on the command-line::
@@ -97,11 +136,11 @@ Building the reference setup
         builds_dir = "/Users/gitlab/builds"  # set this or bugs occur
         cache_dir = "/Users/gitlab/caches"  # this is optional, but desirable
         shell = "bash"
-7. While at the gitlab user, install `Docker for Mac`_.  Ensure to set it up to
+8. While at the gitlab user, install `Docker for Mac`_.  Ensure to set it up to
    start at login.  In "Preferences > Filesystem Sharing", ensure that
    `/var/folders` is included in the list (that is the default location for
    temporary files in macOS).
-8. Reboot the machine. At this point, the gitlab user should be auto-logged and
+9. Reboot the machine. At this point, the gitlab user should be auto-logged and
    the runner process should be executing.  Congratulations, you're done!