"""Set of common tools to aid bundler implementations."""
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
import os
import shutil
import errno
import nbformat
import fnmatch
import glob
def get_file_references(abs_nb_path, version):
"""Gets a list of files referenced either in Markdown fenced code blocks
or in HTML comments from the notebook. Expands patterns expressed in
gitignore syntax (https://git-scm.com/docs/gitignore). Returns the
fully expanded list of filenames relative to the notebook dirname.
Parameters
----------
abs_nb_path: str
Absolute path of the notebook on disk
version: int
Version of the notebook document format to use
Returns
-------
list
Filename strings relative to the notebook path
"""
ref_patterns = get_reference_patterns(abs_nb_path, version)
expanded = expand_references(os.path.dirname(abs_nb_path), ref_patterns)
return expanded
def get_reference_patterns(abs_nb_path, version):
"""Gets a list of reference patterns either in Markdown fenced code blocks
or in HTML comments from the notebook.
Parameters
----------
abs_nb_path: str
Absolute path of the notebook on disk
version: int
Version of the notebook document format to use
Returns
-------
list
Pattern strings from the notebook
"""
notebook = nbformat.read(abs_nb_path, version)
referenced_list = []
for cell in notebook.cells:
references = get_cell_reference_patterns(cell)
if references:
referenced_list = referenced_list + references
return referenced_list
def get_cell_reference_patterns(cell):
'''
Retrieves the list of references from a single notebook cell. Looks for
fenced code blocks or HTML comments in Markdown cells, e.g.,
```
some.csv
foo/
!foo/bar
```
or
<!--associate:
some.csv
foo/
!foo/bar
-->
Parameters
----------
cell: dict
Notebook cell object
Returns
-------
list
Reference patterns found in the cell
'''
referenced = []
# invisible after execution: unrendered HTML comment
if cell.get('cell_type').startswith('markdown') and cell.get('source').startswith('<!--associate:'):
lines = cell.get('source')[len('<!--associate:'):].splitlines()
for line in lines:
if line.startswith('-->'):
break
# Trying to go out of the current directory leads to
# trouble when deploying
if line.find('../') < 0 and not line.startswith('#'):
referenced.append(line)
# visible after execution: rendered as a code element within a pre element
elif cell.get('cell_type').startswith('markdown') and cell.get('source').find('```') >= 0:
source = cell.get('source')
offset = source.find('```')
lines = source[offset + len('```'):].splitlines()
for line in lines:
if line.startswith('```'):
break
# Trying to go out of the current directory leads to
# trouble when deploying
if line.find('../') < 0 and not line.startswith('#'):
referenced.append(line)
# Clean out blank references
return [ref for ref in referenced if ref.strip()]
def expand_references(root_path, references):
"""Expands a set of reference patterns by evaluating them against the
given root directory. Expansions are performed against patterns
expressed in the same manner as in gitignore
(https://git-scm.com/docs/gitignore).
NOTE: Temporarily changes the current working directory when called.
Parameters
----------
root_path: str
Assumed root directory for the patterns
references: list
Reference patterns from get_reference_patterns expressed with
forward-slash directory separators
Returns
-------
list
Filename strings relative to the root path
"""
# Use normpath to convert to platform specific slashes, but be sure
# to retain a trailing slash which normpath pulls off
normalized_references = []
for ref in references:
normalized_ref = os.path.normpath(ref)
# un-normalized separator
if ref.endswith('/'):
normalized_ref += os.sep
normalized_references.append(normalized_ref)
references = normalized_references
globbed = []
negations = []
must_walk = []
for pattern in references:
if pattern and pattern.find(os.sep) < 0:
# simple shell glob
cwd = os.getcwd()
os.chdir(root_path)
if pattern.startswith('!'):
negations = negations + glob.glob(pattern[1:])
else:
globbed = globbed + glob.glob(pattern)
os.chdir(cwd)
elif pattern:
must_walk.append(pattern)
for pattern in must_walk:
pattern_is_negation = pattern.startswith('!')
if pattern_is_negation:
testpattern = pattern[1:]
else:
testpattern = pattern
for root, _, filenames in os.walk(root_path):
for filename in filenames:
joined = os.path.join(root[len(root_path) + 1:], filename)
if testpattern.endswith(os.sep):
if joined.startswith(testpattern):
if pattern_is_negation:
negations.append(joined)
else:
globbed.append(joined)
elif testpattern.find('**') >= 0:
# path wildcard
ends = testpattern.split('**')
if len(ends) == 2:
if joined.startswith(ends[0]) and joined.endswith(ends[1]):
if pattern_is_negation:
negations.append(joined)
else:
globbed.append(joined)
else:
# segments should be respected
if fnmatch.fnmatch(joined, testpattern):
if pattern_is_negation:
negations.append(joined)
else:
globbed.append(joined)
for negated in negations:
try:
globbed.remove(negated)
except ValueError as err:
pass
return set(globbed)
def copy_filelist(src, dst, src_relative_filenames):
"""Copies the given list of files, relative to src, into dst, creating
directories along the way as needed and ignore existence errors.
Skips any files that do not exist. Does not create empty directories
from src in dst.
Parameters
----------
src: str
Root of the source directory
dst: str
Root of the destination directory
src_relative_filenames: list
Filenames relative to src
"""
for filename in src_relative_filenames:
# Only consider the file if it exists in src
if os.path.isfile(os.path.join(src, filename)):
parent_relative = os.path.dirname(filename)
if parent_relative:
# Make sure the parent directory exists
parent_dst = os.path.join(dst, parent_relative)
try:
os.makedirs(parent_dst)
except OSError as exc:
if exc.errno == errno.EEXIST:
pass
else:
raise exc
shutil.copy2(os.path.join(src, filename), os.path.join(dst, filename))