/breezy-debian/trunk

To get this branch, use:
bzr branch https://code.breezy-vcs.org/breezy-debian/trunk
18.1.39 by James Westby
Place the package under the GPL
1
#    util.py -- Utility functions
18.1.43 by James Westby
Correct (C) statements. Add copyright to setup.py
2
#    Copyright (C) 2006 James Westby <jw+debian@jameswestby.net>
286 by James Westby
Improve formatting and documentation
3
#
18.1.39 by James Westby
Place the package under the GPL
4
#    This file is part of bzr-builddeb.
5
#
75.1.2 by Jelmer Vernooij
Fix some typos
6
#    bzr-builddeb is free software; you can redistribute it and/or modify
18.1.39 by James Westby
Place the package under the GPL
7
#    it under the terms of the GNU General Public License as published by
8
#    the Free Software Foundation; either version 2 of the License, or
9
#    (at your option) any later version.
10
#
11
#    bzr-builddeb is distributed in the hope that it will be useful,
12
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
13
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14
#    GNU General Public License for more details.
15
#
16
#    You should have received a copy of the GNU General Public License
17
#    along with bzr-builddeb; if not, write to the Free Software
18
#    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
19
#
18.1.35 by James Westby
Refactored code to use multiple files.
20
750 by Jelmer Vernooij
use relative imports.
21
from __future__ import absolute_import
22
775.1.1 by Jelmer Vernooij
Swap arguments.
23
import errno
1105 by Jelmer Vernooij
Bump required version of Breezy to 3.1.
24
import hashlib
387 by James Westby
Handle no-trees repo in merge-upstream, and subprocess_setup in more places.
25
import signal
18.1.35 by James Westby
Refactored code to use multiple files.
26
import shutil
570.2.7 by Jelmer Vernooij
Factor out extracting of orig tarballs.
27
import subprocess
327.2.7 by James Westby
Move to using SourceDistiller.
28
import tempfile
18.1.35 by James Westby
Refactored code to use multiple files.
29
import os
193.1.3 by Jelmer Vernooij
Add ability to easily build snapshots by using ~bzr in the changelog version.
30
import re
18.1.35 by James Westby
Refactored code to use multiple files.
31
756 by Jelmer Vernooij
Update for breezy, drop support for older versions of bazaar and for debian_bundle.
32
from debian import deb822
33
from debian.changelog import Changelog, ChangelogParseError
1040 by Jelmer Vernooij
Handle NotMachineReadableError.
34
from debian.copyright import Copyright, NotMachineReadableError
18.1.35 by James Westby
Refactored code to use multiple files.
35
750 by Jelmer Vernooij
use relative imports.
36
from ... import (
494.3.8 by Jelmer Vernooij
Move more functionality onto UpstreamSource classes.
37
    bugtracker,
38
    errors,
1023 by Jelmer Vernooij
Initial support on building packages that don't live in the branch
39
    osutils,
494.3.8 by Jelmer Vernooij
Move more functionality onto UpstreamSource classes.
40
    urlutils,
41
    )
801 by Jelmer Vernooij
Initial work on Python 3 compatibility.
42
from ...trace import (
43
    mutter,
44
    warning,
45
    )
750 by Jelmer Vernooij
use relative imports.
46
from ...transport import (
377 by James Westby
Handle redirects in more cases. Thanks Muharem.
47
    do_catching_redirections,
48
    get_transport,
375.2.2 by Muharem Hrnjadovic
Changes:
49
    )
750 by Jelmer Vernooij
use relative imports.
50
from . import (
403 by James Westby
Add the start of a dh_make command.
51
    global_conf,
52
    )
750 by Jelmer Vernooij
use relative imports.
53
from .config import (
486.3.4 by Jelmer Vernooij
Add more tests, raise exception if there is an inconsistency between source format nativity and version nativity.
54
    DebBuildConfig,
55
    BUILD_TYPE_MERGE,
56
    BUILD_TYPE_NATIVE,
57
    BUILD_TYPE_NORMAL,
58
    )
750 by Jelmer Vernooij
use relative imports.
59
from .errors import (
933 by Jelmer Vernooij
Move get_build_architecture to utils.
60
    BzrError,
494.3.8 by Jelmer Vernooij
Move more functionality onto UpstreamSource classes.
61
    AddChangelogError,
62
    InconsistentSourceFormatError,
63
    NoPreviousUpload,
64
    UnableToFindPreviousUpload,
65
    UnparseableChangelog,
66
    )
18.1.35 by James Westby
Refactored code to use multiple files.
67
1093 by Jelmer Vernooij
Move constants to util.py.
68
BUILDDEB_DIR = '.bzr-builddeb'
69
70
NEW_LOCAL_CONF = 'debian/local.conf.local'
71
NEW_CONF = 'debian/bzr-builddeb.conf'
72
DEFAULT_CONF = os.path.join(BUILDDEB_DIR, 'default.conf')
73
LOCAL_CONF = os.path.join(BUILDDEB_DIR, 'local.conf')
74
946 by Jelmer Vernooij
Move MissingChangelogError to util.
75
76
class MissingChangelogError(BzrError):
77
    _fmt = 'Could not find changelog at %(location)s in tree.'
78
79
    def __init__(self, locations):
80
        BzrError.__init__(self, location=locations)
81
82
712 by Jelmer Vernooij
Defer import of distro_info.
83
_DEBIAN_RELEASES = None
84
_UBUNTU_RELEASES = None
85
1007 by Jelmer Vernooij
Some reformatting.
86
712 by Jelmer Vernooij
Defer import of distro_info.
87
def _get_release_names():
714 by Jelmer Vernooij
Fix tests.
88
    global _DEBIAN_RELEASES, _UBUNTU_RELEASES
712 by Jelmer Vernooij
Defer import of distro_info.
89
    try:
90
        from distro_info import DebianDistroInfo, UbuntuDistroInfo
91
    except ImportError:
1007 by Jelmer Vernooij
Some reformatting.
92
        warning(
93
            "distro_info not available. Unable to retrieve current "
712 by Jelmer Vernooij
Defer import of distro_info.
94
            "list of releases.")
95
        _DEBIAN_RELEASES = []
96
        _UBUNTU_RELEASES = []
97
    else:
98
        # distro info is not available
99
        _DEBIAN_RELEASES = DebianDistroInfo().all
100
        _UBUNTU_RELEASES = UbuntuDistroInfo().all
101
102
    _DEBIAN_RELEASES.extend(['stable', 'testing', 'unstable', 'frozen'])
103
104
105
def debian_releases():
106
    if _DEBIAN_RELEASES is None:
107
        _get_release_names()
108
    return _DEBIAN_RELEASES
109
1007 by Jelmer Vernooij
Some reformatting.
110
712 by Jelmer Vernooij
Defer import of distro_info.
111
def ubuntu_releases():
112
    if _UBUNTU_RELEASES is None:
113
        _get_release_names()
114
    return _UBUNTU_RELEASES
115
1007 by Jelmer Vernooij
Some reformatting.
116
451 by James Westby
Add a --package-merge option to builddeb.
117
DEBIAN_POCKETS = ('', '-security', '-proposed-updates', '-backports')
118
UBUNTU_POCKETS = ('', '-proposed', '-updates', '-security', '-backports')
119
120
423.2.1 by John Arbash Meinel
Fix bug #508251, fall back to iso-8859-1 for author info if utf-8 fails.
121
def safe_decode(s):
122
    """Decode a string into a Unicode value."""
1057 by Jelmer Vernooij
Avoid sixish.
123
    if isinstance(s, str):  # Already unicode
801 by Jelmer Vernooij
Initial work on Python 3 compatibility.
124
        mutter('safe_decode() called on an already-decoded string: %r' % (s,))
423.2.1 by John Arbash Meinel
Fix bug #508251, fall back to iso-8859-1 for author info if utf-8 fails.
125
        return s
126
    try:
127
        return s.decode('utf-8')
798 by Jelmer Vernooij
Initial work on python3 support.
128
    except UnicodeDecodeError as e:
1007 by Jelmer Vernooij
Some reformatting.
129
        mutter('safe_decode(%r) falling back to iso-8859-1: %s' % (s, e))
423.2.1 by John Arbash Meinel
Fix bug #508251, fall back to iso-8859-1 for author info if utf-8 fails.
130
        # TODO: Looking at BeautifulSoup it seems to use 'chardet' to try to
131
        #       guess the encoding of a given text stream. We might want to
132
        #       take a closer look at that.
133
        # TODO: Another possibility would be to make the fallback encoding
134
        #       configurable, possibly exposed as a command-line flag, for now,
135
        #       this seems 'good enough'.
136
        return s.decode('iso-8859-1')
137
138
18.1.35 by James Westby
Refactored code to use multiple files.
139
def recursive_copy(fromdir, todir):
286 by James Westby
Improve formatting and documentation
140
    """Copy the contents of fromdir to todir.
141
142
    Like shutil.copytree, but the destination directory must already exist
143
    with this method, rather than not exists for shutil.
144
    """
145
    mutter("Copying %s to %s", fromdir, todir)
146
    for entry in os.listdir(fromdir):
147
        path = os.path.join(fromdir, entry)
148
        if os.path.isdir(path):
149
            tosubdir = os.path.join(todir, entry)
150
            if not os.path.exists(tosubdir):
151
                os.mkdir(tosubdir)
152
            recursive_copy(path, tosubdir)
153
        else:
1056 by Jelmer Vernooij
Don't follow symlinks when exporting directories.
154
            # Python 3 has a follow_symlinks argument to shutil.copy, but
155
            # Python 2 does not...
156
            if os.path.islink(path):
157
                os.symlink(os.readlink(path), os.path.join(todir, entry))
158
            else:
159
                shutil.copy(path, todir)
18.1.35 by James Westby
Refactored code to use multiple files.
160
161
1023 by Jelmer Vernooij
Initial support on building packages that don't live in the branch
162
def find_changelog(t, subpath='', merge=False, max_blocks=1):
286 by James Westby
Improve formatting and documentation
163
    """Find the changelog in the given tree.
164
165
    First looks for 'debian/changelog'. If "merge" is true will also
166
    look for 'changelog'.
167
350 by Jelmer Vernooij
Look past UNRELEASED changelog entries in merge-upstream. Thanks Jelmer.
168
    The returned changelog is created with 'allow_empty_author=True'
169
    as some people do this but still want to build.
170
    'max_blocks' defaults to 1 to try and prevent old broken
1007 by Jelmer Vernooij
Some reformatting.
171
    changelog entries from causing the command to fail.
286 by James Westby
Improve formatting and documentation
172
613.1.1 by Jonathan Riddell
rename meaningless property name 'larstiq' to 'top_level'
173
    "top_level" is a subset of "merge" mode. It indicates that the
286 by James Westby
Improve formatting and documentation
174
    '.bzr' dir is at the same level as 'changelog' etc., rather
175
    than being at the same level as 'debian/'.
176
177
    :param t: the Tree to look in.
178
    :param merge: whether this is a "merge" package.
447.1.1 by Robert Collins
Add import-upstream command which imports an upstream - useful for
179
    :param max_blocks: Number of max_blocks to parse (defaults to 1). Use None
180
        to parse the entire changelog.
613.1.1 by Jonathan Riddell
rename meaningless property name 'larstiq' to 'top_level'
181
    :return: (changelog, top_level) where changelog is the Changelog,
182
        and top_level is a boolean indicating whether the file is
183
        located at 'changelog' (rather than 'debian/changelog') if
184
        merge was given, False otherwise.
286 by James Westby
Improve formatting and documentation
185
    """
613.1.1 by Jonathan Riddell
rename meaningless property name 'larstiq' to 'top_level'
186
    top_level = False
779 by Jelmer Vernooij
Use with statement for locks, use Tree.is_versioned.
187
    with t.lock_read():
1023 by Jelmer Vernooij
Initial support on building packages that don't live in the branch
188
        changelog_file = osutils.pathjoin(subpath, 'debian/changelog')
286 by James Westby
Improve formatting and documentation
189
        if not t.has_filename(changelog_file):
1023 by Jelmer Vernooij
Initial support on building packages that don't live in the branch
190
            checked_files = [changelog_file]
286 by James Westby
Improve formatting and documentation
191
            if merge:
486.3.2 by Jelmer Vernooij
Add a build_type enum to use instead of separate boolean variables split/merge/native.
192
                # Assume LarstiQ's layout (.bzr in debian/)
1023 by Jelmer Vernooij
Initial support on building packages that don't live in the branch
193
                changelog_file = osutils.pathjoin(subpath, 'changelog')
613.1.1 by Jonathan Riddell
rename meaningless property name 'larstiq' to 'top_level'
194
                top_level = True
286 by James Westby
Improve formatting and documentation
195
                if not t.has_filename(changelog_file):
687 by Jelmer Vernooij
Merge improved error message when changelog couldn't be found.
196
                    checked_files.append(changelog_file)
678.3.2 by Jelmer Vernooij
Fix tests.
197
                    changelog_file = None
198
            else:
199
                changelog_file = None
678.3.1 by Jelmer Vernooij
Include full changelog paths that were checked in error message. LP:
200
            if changelog_file is None:
201
                if getattr(t, "abspath", None):
202
                    checked_files = [t.abspath(f) for f in checked_files]
203
                raise MissingChangelogError(" or ".join(checked_files))
1023 by Jelmer Vernooij
Initial support on building packages that don't live in the branch
204
        elif merge and t.has_filename(osutils.pathjoin(subpath, 'changelog')):
613.1.1 by Jonathan Riddell
rename meaningless property name 'larstiq' to 'top_level'
205
            # If it is a "top_level" package and debian is a symlink to
290 by James Westby
Cleanup of the repack tarball code, splitting out the work for each type.
206
            # "." then it will have found debian/changelog. Try and detect
207
            # this.
1023 by Jelmer Vernooij
Initial support on building packages that don't live in the branch
208
            debian_file = osutils.pathjoin(subpath, 'debian')
209
            if (t.is_versioned(debian_file) and
210
                    t.kind(debian_file) == 'symlink' and
211
                    t.get_symlink_target(debian_file) == '.'):
290 by James Westby
Cleanup of the repack tarball code, splitting out the work for each type.
212
                changelog_file = 'changelog'
613.1.1 by Jonathan Riddell
rename meaningless property name 'larstiq' to 'top_level'
213
                top_level = True
290 by James Westby
Cleanup of the repack tarball code, splitting out the work for each type.
214
        mutter("Using '%s' to get package information", changelog_file)
779 by Jelmer Vernooij
Use with statement for locks, use Tree.is_versioned.
215
        if not t.is_versioned(changelog_file):
290 by James Westby
Cleanup of the repack tarball code, splitting out the work for each type.
216
            raise AddChangelogError(changelog_file)
775.1.1 by Jelmer Vernooij
Swap arguments.
217
        contents = t.get_file_text(changelog_file)
104.2.2 by James Westby
* Allow the changelog entries to have no author information.
218
    changelog = Changelog()
327.1.3 by James Westby
Catch and wrap ChangelogParseError to avoid the traceback (LP: #215732)
219
    try:
1007 by Jelmer Vernooij
Some reformatting.
220
        changelog.parse_changelog(
221
            contents, max_blocks=max_blocks, allow_empty_author=True)
798 by Jelmer Vernooij
Initial work on python3 support.
222
    except ChangelogParseError as e:
327.1.3 by James Westby
Catch and wrap ChangelogParseError to avoid the traceback (LP: #215732)
223
        raise UnparseableChangelog(str(e))
613.1.1 by Jonathan Riddell
rename meaningless property name 'larstiq' to 'top_level'
224
    return changelog, top_level
18.1.35 by James Westby
Refactored code to use multiple files.
225
286 by James Westby
Improve formatting and documentation
226
273.1.1 by Colin Watson
Strip imported changelog entries in the same way as debcommit.
227
def strip_changelog_message(changes):
286 by James Westby
Improve formatting and documentation
228
    """Strip a changelog message like debcommit does.
229
230
    Takes a list of changes from a changelog entry and applies a transformation
231
    so the message is well formatted for a commit message.
232
233
    :param changes: a list of lines from the changelog entry
234
    :return: another list of lines with blank lines stripped from the start
1007 by Jelmer Vernooij
Some reformatting.
235
        and the spaces the start of the lines split if there is only one
236
        logical entry.
286 by James Westby
Improve formatting and documentation
237
    """
327.1.28 by James Westby
Tests for strip_changelog_message.
238
    if not changes:
239
        return changes
240
    while changes and changes[-1] == '':
286 by James Westby
Improve formatting and documentation
241
        changes.pop()
327.1.28 by James Westby
Tests for strip_changelog_message.
242
    while changes and changes[0] == '':
286 by James Westby
Improve formatting and documentation
243
        changes.pop(0)
244
245
    whitespace_column_re = re.compile(r'  |\t')
801 by Jelmer Vernooij
Initial work on Python 3 compatibility.
246
    changes = [whitespace_column_re.sub('', line, 1) for line in changes]
286 by James Westby
Improve formatting and documentation
247
248
    leader_re = re.compile(r'[ \t]*[*+-] ')
801 by Jelmer Vernooij
Initial work on Python 3 compatibility.
249
    count = len([l for l in changes if leader_re.match(l)])
286 by James Westby
Improve formatting and documentation
250
    if count == 1:
801 by Jelmer Vernooij
Initial work on Python 3 compatibility.
251
        return [leader_re.sub('', line, 1).lstrip() for line in changes]
286 by James Westby
Improve formatting and documentation
252
    else:
253
        return changes
254
273.1.1 by Colin Watson
Strip imported changelog entries in the same way as debcommit.
255
578.2.7 by Jelmer Vernooij
Support fetching component tarballs.
256
def tarball_name(package, version, component=None, format=None):
286 by James Westby
Improve formatting and documentation
257
    """Return the name of the .orig.tar.gz for the given package and version.
258
259
    :param package: the name of the source package.
260
    :param version: the upstream version of the package.
578.2.7 by Jelmer Vernooij
Support fetching component tarballs.
261
    :param component: Component name (None for base)
405.1.1 by James Westby
Support for building using .tar.bz2|lzma.
262
    :param format: the format for the tarball. If None then 'gz' will be
612.1.1 by Jelmer Vernooij
.tar.lzma -> .tar.xz
263
         used. You probably want on of 'gz', 'bz2', 'lzma' or 'xz'.
286 by James Westby
Improve formatting and documentation
264
    :return: a string that is the name of the upstream tarball to use.
265
    """
405.1.1 by James Westby
Support for building using .tar.bz2|lzma.
266
    if format is None:
267
        format = 'gz'
578.2.7 by Jelmer Vernooij
Support fetching component tarballs.
268
    name = "%s_%s.orig" % (package, str(version))
269
    if component is not None:
270
        name += "-" + component
271
    return "%s.tar.%s" % (name, format)
18.1.35 by James Westby
Refactored code to use multiple files.
272
327.1.54 by James Westby
Make --export-upstream work again by not trying to use a removed attribute
273
294 by James Westby
Factor out the code for spotting "debian" and "ubuntu" in lookup_distribution.
274
def suite_to_distribution(suite):
286 by James Westby
Improve formatting and documentation
275
    """Infer the distribution from a suite.
276
277
    When passed the name of a suite (anything in the distributions field of
278
    a changelog) it will infer the distribution from that (i.e. Debian or
279
    Ubuntu).
280
294 by James Westby
Factor out the code for spotting "debian" and "ubuntu" in lookup_distribution.
281
    :param suite: the string containing the suite
485.3.1 by Jelmer Vernooij
Add some docstrings, fix typo in temporary file name.
282
    :return: "debian", "ubuntu", or None if the distribution couldn't be
283
        inferred.
286 by James Westby
Improve formatting and documentation
284
    """
712 by Jelmer Vernooij
Defer import of distro_info.
285
    all_debian = [r + t for r in debian_releases() for t in DEBIAN_POCKETS]
286
    all_ubuntu = [r + t for r in ubuntu_releases() for t in UBUNTU_POCKETS]
294 by James Westby
Factor out the code for spotting "debian" and "ubuntu" in lookup_distribution.
287
    if suite in all_debian:
286 by James Westby
Improve formatting and documentation
288
        return "debian"
294 by James Westby
Factor out the code for spotting "debian" and "ubuntu" in lookup_distribution.
289
    if suite in all_ubuntu:
286 by James Westby
Improve formatting and documentation
290
        return "ubuntu"
291
    return None
294 by James Westby
Factor out the code for spotting "debian" and "ubuntu" in lookup_distribution.
292
295 by James Westby
Spot any case of distribution name in lookup_distribution.
293
294 by James Westby
Factor out the code for spotting "debian" and "ubuntu" in lookup_distribution.
294
def lookup_distribution(distribution_or_suite):
295 by James Westby
Spot any case of distribution name in lookup_distribution.
295
    """Get the distribution name based on a distribtion or suite name.
296
297
    :param distribution_or_suite: a string that is either the name of
298
        a distribution or a suite.
299
    :return: a string with a distribution name or None.
300
    """
301
    if distribution_or_suite.lower() in ("debian", "ubuntu"):
302
        return distribution_or_suite.lower()
294 by James Westby
Factor out the code for spotting "debian" and "ubuntu" in lookup_distribution.
303
    return suite_to_distribution(distribution_or_suite)
327.2.7 by James Westby
Move to using SourceDistiller.
304
305
447.1.1 by Robert Collins
Add import-upstream command which imports an upstream - useful for
306
def md5sum_filename(filename):
485.3.1 by Jelmer Vernooij
Add some docstrings, fix typo in temporary file name.
307
    """Calculate the md5sum of a file by name.
308
309
    :param filename: Path of the file to checksum
310
    :return: MD5 Checksum as hex digest
311
    """
1105 by Jelmer Vernooij
Bump required version of Breezy to 3.1.
312
    m = hashlib.md5()
447.1.1 by Robert Collins
Add import-upstream command which imports an upstream - useful for
313
    f = open(filename, 'rb')
314
    try:
315
        for line in f:
316
            m.update(line)
317
    finally:
318
        f.close()
319
    return m.hexdigest()
320
321
327.2.7 by James Westby
Move to using SourceDistiller.
322
def move_file_if_different(source, target, md5sum):
1007 by Jelmer Vernooij
Some reformatting.
323
    """Overwrite a file if its new contents would be different from the current
324
    contents.
485.3.1 by Jelmer Vernooij
Add some docstrings, fix typo in temporary file name.
325
326
    :param source: Path of the source file
327
    :param target: Path of the target file
328
    :param md5sum: MD5Sum (as hex digest) of the source file
329
    """
327.2.7 by James Westby
Move to using SourceDistiller.
330
    if os.path.exists(target):
331
        if os.path.samefile(source, target):
332
            return
447.1.1 by Robert Collins
Add import-upstream command which imports an upstream - useful for
333
        t_md5sum = md5sum_filename(target)
334
        if t_md5sum == md5sum:
327.2.7 by James Westby
Move to using SourceDistiller.
335
            return
336
    shutil.move(source, target)
337
338
339
def write_if_different(contents, target):
1007 by Jelmer Vernooij
Some reformatting.
340
    """(Over)write a file with `contents` if they are different from its
341
    current content.
485.3.1 by Jelmer Vernooij
Add some docstrings, fix typo in temporary file name.
342
343
    :param contents: The contents to write, as a string
344
    :param target: Path of the target file
345
    """
1105 by Jelmer Vernooij
Bump required version of Breezy to 3.1.
346
    md5sum = hashlib.md5()
327.2.7 by James Westby
Move to using SourceDistiller.
347
    md5sum.update(contents)
348
    fd, temp_path = tempfile.mkstemp("builddeb-rename-")
798 by Jelmer Vernooij
Initial work on python3 support.
349
    fobj = os.fdopen(fd, "wb")
327.2.7 by James Westby
Move to using SourceDistiller.
350
    try:
351
        try:
327.1.24 by James Westby
Open the file descriptors to get file objects.
352
            fobj.write(contents)
327.2.7 by James Westby
Move to using SourceDistiller.
353
        finally:
327.1.24 by James Westby
Open the file descriptors to get file objects.
354
            fobj.close()
327.2.7 by James Westby
Move to using SourceDistiller.
355
        move_file_if_different(temp_path, target, md5sum.hexdigest())
356
    finally:
357
        if os.path.exists(temp_path):
358
            os.unlink(temp_path)
359
360
361
def _download_part(name, base_transport, target_dir, md5sum):
362
    part_base_dir, part_path = urlutils.split(name)
363
    f_t = base_transport
364
    if part_base_dir != '':
365
        f_t = base_transport.clone(part_base_dir)
366
    f_f = f_t.get(part_path)
367
    try:
368
        target_path = os.path.join(target_dir, part_path)
485.3.1 by Jelmer Vernooij
Add some docstrings, fix typo in temporary file name.
369
        fd, temp_path = tempfile.mkstemp(prefix="builddeb-")
327.1.24 by James Westby
Open the file descriptors to get file objects.
370
        fobj = os.fdopen(fd, "wb")
327.2.7 by James Westby
Move to using SourceDistiller.
371
        try:
372
            try:
327.1.24 by James Westby
Open the file descriptors to get file objects.
373
                shutil.copyfileobj(f_f, fobj)
327.2.7 by James Westby
Move to using SourceDistiller.
374
            finally:
327.1.24 by James Westby
Open the file descriptors to get file objects.
375
                fobj.close()
327.2.7 by James Westby
Move to using SourceDistiller.
376
            move_file_if_different(temp_path, target_path, md5sum)
377
        finally:
378
            if os.path.exists(temp_path):
379
                os.unlink(temp_path)
380
    finally:
381
        f_f.close()
382
383
485.3.1 by Jelmer Vernooij
Add some docstrings, fix typo in temporary file name.
384
def open_file(url):
385
    """Open a file from a URL.
386
387
    :param url: URL to open
388
    :return: A file-like object.
389
    """
390
    filename, transport = open_transport(url)
375.2.2 by Muharem Hrnjadovic
Changes:
391
    return open_file_via_transport(filename, transport)
392
393
394
def open_transport(path):
562.1.1 by Max Bowsher
Mend inconsistencies where different indent widths are used at different points
395
    """Obtain an appropriate transport instance for the given path."""
396
    base_dir, path = urlutils.split(path)
397
    transport = get_transport(base_dir)
398
    return (path, transport)
375.2.2 by Muharem Hrnjadovic
Changes:
399
400
401
def open_file_via_transport(filename, transport):
562.1.1 by Max Bowsher
Mend inconsistencies where different indent widths are used at different points
402
    """Open a file using the transport, follow redirects as necessary."""
403
    def open_file(transport):
404
        return transport.get(filename)
1007 by Jelmer Vernooij
Some reformatting.
405
562.1.1 by Max Bowsher
Mend inconsistencies where different indent widths are used at different points
406
    def follow_redirection(transport, e, redirection_notice):
407
        mutter(redirection_notice)
408
        _filename, redirected_transport = open_transport(e.target)
409
        return redirected_transport
375.2.2 by Muharem Hrnjadovic
Changes:
410
562.1.1 by Max Bowsher
Mend inconsistencies where different indent widths are used at different points
411
    result = do_catching_redirections(open_file, transport, follow_redirection)
412
    return result
375.2.2 by Muharem Hrnjadovic
Changes:
413
414
327.2.7 by James Westby
Move to using SourceDistiller.
415
def _dget(cls, dsc_location, target_dir):
877 by Jelmer Vernooij
Factor out obtaining distiller.
416
    """Copy all files referenced by a .dsc file.
417
418
    Args:
419
      cls: Parser class
420
      dsc_location: Source file location
421
      target_dir: Target directory
422
    Return:
423
      path to target source file
424
    """
327.1.31 by James Westby
Add tests for dget and dget_changes.
425
    if not os.path.isdir(target_dir):
426
        raise errors.NotADirectory(target_dir)
375.2.2 by Muharem Hrnjadovic
Changes:
427
    path, dsc_t = open_transport(dsc_location)
1044 by Jelmer Vernooij
Close file handle.
428
    with open_file_via_transport(path, dsc_t) as f:
429
        dsc_contents = f.read()
327.2.7 by James Westby
Move to using SourceDistiller.
430
    dsc = cls(dsc_contents)
431
    for file_details in dsc['files']:
432
        name = file_details['name']
433
        _download_part(name, dsc_t, target_dir, file_details['md5sum'])
341 by James Westby
Add karmic.
434
    target_file = os.path.join(target_dir, path)
435
    write_if_different(dsc_contents, target_file)
436
    return target_file
327.2.7 by James Westby
Move to using SourceDistiller.
437
438
439
def dget(dsc_location, target_dir):
440
    return _dget(deb822.Dsc, dsc_location, target_dir)
441
442
443
def dget_changes(changes_location, target_dir):
444
    return _dget(deb822.Changes, changes_location, target_dir)
327.1.33 by James Westby
Add tests for UpstreamProvider.
445
446
447
def get_parent_dir(target):
448
    parent = os.path.dirname(target)
449
    if os.path.basename(target) == '':
450
        parent = os.path.dirname(parent)
451
    return parent
332 by James Westby
Refactor extracting commit information from the changelog.
452
453
334 by James Westby
Look up other distro bugs via launchpad for LP and Closes.
454
def find_bugs_fixed(changes, branch, _lplib=None):
485.3.1 by Jelmer Vernooij
Add some docstrings, fix typo in temporary file name.
455
    """Find the bugs marked fixed in a changelog entry.
456
561.2.5 by Jonathan Riddell
improve method doc
457
    :param changes: A list of the contents of the changelog entry.
485.3.1 by Jelmer Vernooij
Add some docstrings, fix typo in temporary file name.
458
    :param branch: Bazaar branch associated with the package
1007 by Jelmer Vernooij
Some reformatting.
459
    :return: String with bugs closed, as appropriate for a Bazaar "bugs"
460
        revision property.
485.3.1 by Jelmer Vernooij
Add some docstrings, fix typo in temporary file name.
461
    """
334 by James Westby
Look up other distro bugs via launchpad for LP and Closes.
462
    if _lplib is None:
750 by Jelmer Vernooij
use relative imports.
463
        from . import launchpad as _lplib
332 by James Westby
Refactor extracting commit information from the changelog.
464
    bugs = []
465
    for change in changes:
1007 by Jelmer Vernooij
Some reformatting.
466
        for match in re.finditer(
467
                "closes:\\s*(?:bug)?\\#?\\s?\\d+"
846 by Jelmer Vernooij
Fix some backslash escaping.
468
                "(?:,\\s*(?:bug)?\\#?\\s?\\d+)*", change,
332 by James Westby
Refactor extracting commit information from the changelog.
469
                re.IGNORECASE):
470
            closes_list = match.group(0)
846 by Jelmer Vernooij
Fix some backslash escaping.
471
            for match in re.finditer("\\d+", closes_list):
485.3.1 by Jelmer Vernooij
Add some docstrings, fix typo in temporary file name.
472
                bug_url = bugtracker.get_bug_url("deb", branch, match.group(0))
332 by James Westby
Refactor extracting commit information from the changelog.
473
                bugs.append(bug_url + " fixed")
334 by James Westby
Look up other distro bugs via launchpad for LP and Closes.
474
                lp_bugs = _lplib.ubuntu_bugs_for_debian_bug(match.group(0))
475
                if len(lp_bugs) == 1:
485.3.1 by Jelmer Vernooij
Add some docstrings, fix typo in temporary file name.
476
                    bug_url = bugtracker.get_bug_url("lp", branch, lp_bugs[0])
334 by James Westby
Look up other distro bugs via launchpad for LP and Closes.
477
                    bugs.append(bug_url + " fixed")
1007 by Jelmer Vernooij
Some reformatting.
478
        for match in re.finditer(
479
                "lp:\\s+\\#\\d+(?:,\\s*\\#\\d+)*", change, re.IGNORECASE):
332 by James Westby
Refactor extracting commit information from the changelog.
480
            closes_list = match.group(0)
846 by Jelmer Vernooij
Fix some backslash escaping.
481
            for match in re.finditer("\\d+", closes_list):
561.3.2 by Jonathan Riddell
actually this is fine
482
                bug_url = bugtracker.get_bug_url("lp", branch, match.group(0))
332 by James Westby
Refactor extracting commit information from the changelog.
483
                bugs.append(bug_url + " fixed")
561.3.2 by Jonathan Riddell
actually this is fine
484
                deb_bugs = _lplib.debian_bugs_for_ubuntu_bug(match.group(0))
334 by James Westby
Look up other distro bugs via launchpad for LP and Closes.
485
                if len(deb_bugs) == 1:
1007 by Jelmer Vernooij
Some reformatting.
486
                    bug_url = bugtracker.get_bug_url(
487
                        "deb", branch, deb_bugs[0])
334 by James Westby
Look up other distro bugs via launchpad for LP and Closes.
488
                    bugs.append(bug_url + " fixed")
332 by James Westby
Refactor extracting commit information from the changelog.
489
    return bugs
490
491
492
def find_extra_authors(changes):
485.3.1 by Jelmer Vernooij
Add some docstrings, fix typo in temporary file name.
493
    """Find additional authors from a changelog entry.
494
495
    :return: List of fullnames of additional authors, without e-mail address.
496
    """
423.2.1 by John Arbash Meinel
Fix bug #508251, fall back to iso-8859-1 for author info if utf-8 fails.
497
    extra_author_re = re.compile(r"\s*\[([^\]]+)]\s*")
332 by James Westby
Refactor extracting commit information from the changelog.
498
    authors = []
499
    for change in changes:
500
        # Parse out any extra authors.
423.2.1 by John Arbash Meinel
Fix bug #508251, fall back to iso-8859-1 for author info if utf-8 fails.
501
        match = extra_author_re.match(change)
332 by James Westby
Refactor extracting commit information from the changelog.
502
        if match is not None:
423.2.1 by John Arbash Meinel
Fix bug #508251, fall back to iso-8859-1 for author info if utf-8 fails.
503
            new_author = safe_decode(match.group(1).strip())
332 by James Westby
Refactor extracting commit information from the changelog.
504
            already_included = False
505
            for author in authors:
506
                if author.startswith(new_author):
507
                    already_included = True
508
                    break
509
            if not already_included:
391 by James Westby
Fix the encoding issues around things taken from the changelog.
510
                authors.append(new_author)
332 by James Westby
Refactor extracting commit information from the changelog.
511
    return authors
512
513
514
def find_thanks(changes):
485.3.1 by Jelmer Vernooij
Add some docstrings, fix typo in temporary file name.
515
    """Find all people thanked in a changelog entry.
516
517
    :param changes: String with the contents of the changelog entry
518
    :return: List of people thanked, optionally including email address.
519
    """
1007 by Jelmer Vernooij
Some reformatting.
520
    thanks_re = re.compile(
521
        r"[tT]hank(?:(?:s)|(?:you))(?:\s*to)?"
522
        "((?:\\s+(?:(?:\\w\\.)|(?:\\w+(?:-\\w+)*)))+"
523
        "(?:\\s+<[^@>]+@[^@>]+>)?)",
524
        re.UNICODE)
332 by James Westby
Refactor extracting commit information from the changelog.
525
    thanks = []
423.2.1 by John Arbash Meinel
Fix bug #508251, fall back to iso-8859-1 for author info if utf-8 fails.
526
    changes_str = safe_decode(" ".join(changes))
332 by James Westby
Refactor extracting commit information from the changelog.
527
    for match in thanks_re.finditer(changes_str):
528
        if thanks is None:
529
            thanks = []
530
        thanks_str = match.group(1).strip()
531
        thanks_str = re.sub(r"\s+", " ", thanks_str)
391 by James Westby
Fix the encoding issues around things taken from the changelog.
532
        thanks.append(thanks_str)
332 by James Westby
Refactor extracting commit information from the changelog.
533
    return thanks
534
535
334 by James Westby
Look up other distro bugs via launchpad for LP and Closes.
536
def get_commit_info_from_changelog(changelog, branch, _lplib=None):
332 by James Westby
Refactor extracting commit information from the changelog.
537
    """Retrieves the messages from the last section of debian/changelog.
538
539
    Reads the latest stanza of debian/changelog and returns the
540
    text of the changes in that section. It also returns other
541
    information about the change, including the authors of the change,
542
    anyone that is thanked, and the bugs that are declared fixed by it.
543
544
    :return: a tuple (message, authors, thanks, bugs). message is the
545
        commit message that should be used. authors is a list of strings,
546
        with those that contributed to the change, thanks is a list
547
        of string, with those who were thanked in the changelog entry.
548
        bugs is a list of bug URLs like for --fixes.
549
        If the information is not available then any can be None.
550
    """
551
    message = None
552
    authors = []
553
    thanks = []
554
    bugs = []
555
    if changelog._blocks:
556
        block = changelog._blocks[0]
423.2.1 by John Arbash Meinel
Fix bug #508251, fall back to iso-8859-1 for author info if utf-8 fails.
557
        authors = [safe_decode(block.author)]
332 by James Westby
Refactor extracting commit information from the changelog.
558
        changes = strip_changelog_message(block.changes())
559
        authors += find_extra_authors(changes)
334 by James Westby
Look up other distro bugs via launchpad for LP and Closes.
560
        bugs = find_bugs_fixed(changes, branch, _lplib=_lplib)
332 by James Westby
Refactor extracting commit information from the changelog.
561
        thanks = find_thanks(changes)
423.2.2 by John Arbash Meinel
Change the find_thanks regex to handle opening unicode chars.
562
        message = safe_decode("\n".join(changes).replace("\r", ""))
332 by James Westby
Refactor extracting commit information from the changelog.
563
    return (message, authors, thanks, bugs)
350 by Jelmer Vernooij
Look past UNRELEASED changelog entries in merge-upstream. Thanks Jelmer.
564
565
566
def find_last_distribution(changelog):
567
    """Find the last changelog that was used in a changelog.
568
569
    This will skip stanzas with the 'UNRELEASED' distribution.
448 by James Westby
Add import-upstream command. Thanks Rob.
570
350 by Jelmer Vernooij
Look past UNRELEASED changelog entries in merge-upstream. Thanks Jelmer.
571
    :param changelog: Changelog to analyze
572
    """
573
    for block in changelog._blocks:
574
        distribution = block.distributions.split(" ")[0]
575
        if distribution != "UNRELEASED":
576
            return distribution
577
    return None
387 by James Westby
Handle no-trees repo in merge-upstream, and subprocess_setup in more places.
578
579
580
def subprocess_setup():
581
    # Python installs a SIGPIPE handler by default. This is usually not what
582
    # non-Python subprocesses expect.
583
    # Many, many thanks to Colin Watson
584
    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
403 by James Westby
Add the start of a dh_make command.
585
586
1023 by Jelmer Vernooij
Initial support on building packages that don't live in the branch
587
def debuild_config(tree, subpath):
403 by James Westby
Add the start of a dh_make command.
588
    """Obtain the Debuild configuration object.
589
590
    :param tree: A Tree object, can be a WorkingTree or RevisionTree.
591
    """
592
    config_files = []
593
    user_config = None
1095 by Jelmer Vernooij
look for configuration in subpaths.
594
    if subpath in ('.', None):
595
        subpath = ''
596
    new_local_conf = osutils.pathjoin(subpath, NEW_LOCAL_CONF)
597
    local_conf = osutils.pathjoin(subpath, LOCAL_CONF)
598
    new_conf = osutils.pathjoin(subpath, NEW_CONF)
599
    default_conf = osutils.pathjoin(subpath, DEFAULT_CONF)
600
601
    if tree.has_filename(new_local_conf):
602
        if not tree.is_versioned(new_local_conf):
603
            config_files.append(
604
                (tree.get_file(new_local_conf), True, "local.conf"))
605
        else:
606
            warning('Not using configuration from %s as it is versioned.',
607
                    new_local_conf)
608
    if tree.has_filename(local_conf):
609
        if not tree.is_versioned(local_conf):
610
            config_files.append(
611
                (tree.get_file(local_conf), True, "local.conf"))
612
        else:
613
            warning('Not using configuration from %s as it is versioned.',
614
                    local_conf)
467 by James Westby
Determine Bazaar home directory using bzrlib to prevent test isolation issues.
615
    config_files.append((global_conf(), True))
616
    user_config = global_conf()
1095 by Jelmer Vernooij
look for configuration in subpaths.
617
    if tree.is_versioned(new_conf):
618
        config_files.append(
619
            (tree.get_file(new_conf), False, "bzr-builddeb.conf"))
620
    if tree.is_versioned(default_conf):
621
        config_files.append(
622
            (tree.get_file(default_conf), False, "default.conf"))
403 by James Westby
Add the start of a dh_make command.
623
    config = DebBuildConfig(config_files, tree=tree)
624
    config.set_user_config(user_config)
625
    return config
432.1.2 by Jelmer Vernooij
Add convenience function for using export.
626
627
1023 by Jelmer Vernooij
Initial support on building packages that don't live in the branch
628
def find_previous_upload(tree, subpath, merge=False):
458.1.1 by James Westby
Re-parse the changelog if --package-merge is used so we can see all blocks.
629
    """Given a tree, find the previous upload to the distribution.
451 by James Westby
Add a --package-merge option to builddeb.
630
631
    When e.g. Ubuntu merges from Debian they want to build with
632
    -vPREV_VERSION. Here's where we find that previous version.
633
634
    We look at the last changelog entry and find the upload target.
635
    We then search backwards until we find the same target. That's
636
    the previous version that we return.
637
638
    We require there to be a previous version, otherwise we throw
639
    an error.
640
641
    It's not a simple string comparison to find the same target in
642
    a previous version, as we should consider old series in e.g.
643
    Ubuntu.
644
    """
458.1.1 by James Westby
Re-parse the changelog if --package-merge is used so we can see all blocks.
645
    try:
1023 by Jelmer Vernooij
Initial support on building packages that don't live in the branch
646
        cl, top_level = find_changelog(tree, subpath, merge, max_blocks=None)
458.1.1 by James Westby
Re-parse the changelog if --package-merge is used so we can see all blocks.
647
    except UnparseableChangelog:
648
        raise UnableToFindPreviousUpload()
880 by Jelmer Vernooij
Make changelog_find_previous_upload public.
649
    return changelog_find_previous_upload(cl)
650
651
652
def changelog_find_previous_upload(cl):
485.3.1 by Jelmer Vernooij
Add some docstrings, fix typo in temporary file name.
653
    """Find the version of the previous upload.
654
655
    :param cl: Changelog object
656
    :return: Version object for the previous upload
657
    :raise NoPreviousUpload: Raised when there is no previous upload
658
    """
526.1.1 by Jelmer Vernooij
When building with distribution set to UNRELEASED, assume the build is targetted at the same distribution as the build before.
659
    current_target = find_last_distribution(cl)
712 by Jelmer Vernooij
Defer import of distro_info.
660
    all_debian = [r + t for r in debian_releases() for t in DEBIAN_POCKETS]
661
    all_ubuntu = [r + t for r in ubuntu_releases() for t in UBUNTU_POCKETS]
451 by James Westby
Add a --package-merge option to builddeb.
662
    if current_target in all_debian:
663
        match_targets = (current_target,)
664
    elif current_target in all_ubuntu:
712 by Jelmer Vernooij
Defer import of distro_info.
665
        match_targets = ubuntu_releases()
451 by James Westby
Add a --package-merge option to builddeb.
666
        if "-" in current_target:
1007 by Jelmer Vernooij
Some reformatting.
667
            match_targets += tuple(
668
                [current_target.split("-", 1)[0] + t for t in UBUNTU_POCKETS])
451 by James Westby
Add a --package-merge option to builddeb.
669
    else:
545.1.1 by Max Bowsher
Do not error if the top changelog entry is not a recognized Debian or Ubuntu
670
        # If we do not recognize the current target in order to apply special
671
        # rules to it, then just assume that only previous uploads to exactly
672
        # the same target count.
673
        match_targets = (current_target,)
526.1.1 by Jelmer Vernooij
When building with distribution set to UNRELEASED, assume the build is targetted at the same distribution as the build before.
674
    for block in cl._blocks[1:]:
451 by James Westby
Add a --package-merge option to builddeb.
675
        if block.distributions.split(" ")[0] in match_targets:
676
            return block.version
677
    raise NoPreviousUpload(current_target)
459.2.2 by Jelmer Vernooij
Allow .bzr-builddeb to exist in a tree in merge mode.
678
679
1023 by Jelmer Vernooij
Initial support on building packages that don't live in the branch
680
def tree_contains_upstream_source(tree, subpath=''):
459.2.2 by Jelmer Vernooij
Allow .bzr-builddeb to exist in a tree in merge mode.
681
    """Guess if the specified tree contains the upstream source.
485.3.1 by Jelmer Vernooij
Add some docstrings, fix typo in temporary file name.
682
459.2.2 by Jelmer Vernooij
Allow .bzr-builddeb to exist in a tree in merge mode.
683
    :param tree: A RevisionTree.
684
    :return: Boolean indicating whether or not the tree contains the upstream
591.1.3 by Jelmer Vernooij
When the tree is empty, include the upstream source.
685
        source. None if the tree is empty
459.2.2 by Jelmer Vernooij
Allow .bzr-builddeb to exist in a tree in merge mode.
686
    """
595.1.1 by Jelmer Vernooij
Add tests for tree_contains_upstream_source.
687
    present_files = set(
1023 by Jelmer Vernooij
Initial support on building packages that don't live in the branch
688
        [f[0] for f in tree.list_files(recursive=False, from_dir=subpath)
595.1.1 by Jelmer Vernooij
Add tests for tree_contains_upstream_source.
689
         if f[1] == 'V'])
591.1.3 by Jelmer Vernooij
When the tree is empty, include the upstream source.
690
    if len(present_files) == 0:
691
        return None
734.1.1 by Jelmer Vernooij
Don't consider .gitignore when looking for upstream sources.
692
    packaging_files = frozenset([
693
        "debian", ".bzr-builddeb", ".bzrignore", ".gitignore"])
459.2.2 by Jelmer Vernooij
Allow .bzr-builddeb to exist in a tree in merge mode.
694
    return (len(present_files - packaging_files) > 0)
486.3.3 by Jelmer Vernooij
Automatically use debian/source/format if package is native. Closes:
695
696
1023 by Jelmer Vernooij
Initial support on building packages that don't live in the branch
697
def tree_get_source_format(tree, subpath=''):
486.3.3 by Jelmer Vernooij
Automatically use debian/source/format if package is native. Closes:
698
    """Retrieve the source format name from a package.
699
700
    :param path: Path to the package
701
    :return: String with package format
702
    """
1023 by Jelmer Vernooij
Initial support on building packages that don't live in the branch
703
    filename = osutils.pathjoin(subpath, "debian/source/format")
775.1.1 by Jelmer Vernooij
Swap arguments.
704
    try:
705
        text = tree.get_file_text(filename)
798 by Jelmer Vernooij
Initial work on python3 support.
706
    except IOError as e:
707
        if e.errno == errno.ENOENT:
775.1.1 by Jelmer Vernooij
Swap arguments.
708
            return FORMAT_1_0
709
        raise
710
    except errors.NoSuchFile:
684 by Jelmer Vernooij
Automatically apply patches for 3.0 (quilt) packages in 'bzr bd-do'
711
        return FORMAT_1_0
800 by Jelmer Vernooij
Return source format as string.
712
    return text.strip().decode('ascii')
486.3.4 by Jelmer Vernooij
Add more tests, raise exception if there is an inconsistency between source format nativity and version nativity.
713
714
506.1.5 by Jelmer Vernooij
Add constant for format strings.
715
FORMAT_1_0 = "1.0"
716
FORMAT_3_0_QUILT = "3.0 (quilt)"
717
FORMAT_3_0_NATIVE = "3.0 (native)"
718
719
NATIVE_SOURCE_FORMATS = [FORMAT_3_0_NATIVE]
720
NORMAL_SOURCE_FORMATS = [FORMAT_3_0_QUILT]
486.3.4 by Jelmer Vernooij
Add more tests, raise exception if there is an inconsistency between source format nativity and version nativity.
721
722
1023 by Jelmer Vernooij
Initial support on building packages that don't live in the branch
723
def guess_build_type(tree, version, subpath='', contains_upstream_source=True):
486.3.4 by Jelmer Vernooij
Add more tests, raise exception if there is an inconsistency between source format nativity and version nativity.
724
    """Guess the build type based on the contents of a tree.
725
726
    :param tree: A `Tree` object.
727
    :param version: `Version` of the upload.
1007 by Jelmer Vernooij
Some reformatting.
728
    :param contains_upstream_source: Whether this branch contains the upstream
729
        source.
486.3.4 by Jelmer Vernooij
Add more tests, raise exception if there is an inconsistency between source format nativity and version nativity.
730
    :return: A build_type value.
731
    """
1023 by Jelmer Vernooij
Initial support on building packages that don't live in the branch
732
    source_format = tree_get_source_format(tree, subpath)
486.3.4 by Jelmer Vernooij
Add more tests, raise exception if there is an inconsistency between source format nativity and version nativity.
733
    if source_format in NATIVE_SOURCE_FORMATS:
734
        format_native = True
735
    elif source_format in NORMAL_SOURCE_FORMATS:
736
        format_native = False
737
    else:
738
        format_native = None
739
740
    # If the package doesn't have a debian revision then it must be native.
741
    if version is not None:
526.3.1 by Max Bowsher
Use Version.debian_version not Version.debian_revision - the latter requires python-debian 0.1.15 or better.
742
        version_native = (not version.debian_version)
486.3.4 by Jelmer Vernooij
Add more tests, raise exception if there is an inconsistency between source format nativity and version nativity.
743
    else:
744
        version_native = None
745
746
    if type(version_native) is bool and type(format_native) is bool:
747
        if version_native != format_native:
748
            raise InconsistentSourceFormatError(version_native, format_native)
749
750
    if version_native or format_native:
751
        return BUILD_TYPE_NATIVE
1007 by Jelmer Vernooij
Some reformatting.
752
    if contains_upstream_source is False:
486.3.4 by Jelmer Vernooij
Add more tests, raise exception if there is an inconsistency between source format nativity and version nativity.
753
        # Default to merge mode if there's only a debian/ directory
754
        return BUILD_TYPE_MERGE
755
    else:
756
        return BUILD_TYPE_NORMAL
494.3.8 by Jelmer Vernooij
Move more functionality onto UpstreamSource classes.
757
758
570.2.17 by Jelmer Vernooij
Pass along component with tarballs.
759
def component_from_orig_tarball(tarball_filename, package, version):
760
    tarball_filename = os.path.basename(tarball_filename)
761
    prefix = "%s_%s.orig" % (package, version)
762
    if not tarball_filename.startswith(prefix):
763
        raise ValueError(
764
            "invalid orig tarball file %s does not have expected prefix %s" % (
765
                tarball_filename, prefix))
766
    base = tarball_filename[len(prefix):]
612.1.1 by Jelmer Vernooij
.tar.lzma -> .tar.xz
767
    for ext in (".tar.gz", ".tar.bz2", ".tar.lzma", ".tar.xz"):
570.2.17 by Jelmer Vernooij
Pass along component with tarballs.
768
        if tarball_filename.endswith(ext):
769
            base = base[:-len(ext)]
770
            break
771
    else:
772
        raise ValueError(
773
            "orig tarball file %s has unknown extension" % tarball_filename)
774
    if base == "":
775
        return None
776
    elif base[0] == "-":
777
        # Extra component
778
        return base[1:]
779
    else:
1007 by Jelmer Vernooij
Some reformatting.
780
        raise ValueError(
781
                "Invalid extra characters in tarball filename %s" %
782
                tarball_filename)
783
784
1080 by Jelmer Vernooij
Move TarFailed.
785
class TarFailed(BzrError):
1085 by Jelmer Vernooij
Support falling back to strip-components=0 when the orig tarball
786
    _fmt = ("There was an error executing tar to %(operation)s %(tarball)s: "
787
            "%(error)s.")
1080 by Jelmer Vernooij
Move TarFailed.
788
1084 by Jelmer Vernooij
Some more reformatting + extra error messages.
789
    def __init__(self, operation, tarball, error):
1085 by Jelmer Vernooij
Support falling back to strip-components=0 when the orig tarball
790
        BzrError.__init__(
791
            self, operation=operation, tarball=tarball, error=error)
792
793
794
def needs_strip_components(tf):
795
    top_level_directories = set()
796
    for name in tf.getnames():
797
        top_level_directories.add(name.split('/')[0])
798
    return len(top_level_directories) == 1
1080 by Jelmer Vernooij
Move TarFailed.
799
800
1007 by Jelmer Vernooij
Some reformatting.
801
def extract_orig_tarball(tarball_filename, component, target,
802
                         strip_components=None):
570.2.16 by Jelmer Vernooij
Extract extract_orig_tarballs
803
    """Extract an orig tarball.
561.1.1 by Jelmer Vernooij
Move pristine tar upstream to upstream/pristinetar.
804
570.2.16 by Jelmer Vernooij
Extract extract_orig_tarballs
805
    :param tarball: Path to the tarball
806
    :param component: Component name (or None for top-level)
807
    :param target: Target path
570.2.7 by Jelmer Vernooij
Factor out extracting of orig tarballs.
808
    """
1085 by Jelmer Vernooij
Support falling back to strip-components=0 when the orig tarball
809
    from tarfile import TarFile
570.2.7 by Jelmer Vernooij
Factor out extracting of orig tarballs.
810
    tar_args = ["tar"]
811
    if tarball_filename.endswith(".tar.bz2"):
812
        tar_args.append('xjf')
1085 by Jelmer Vernooij
Support falling back to strip-components=0 when the orig tarball
813
        tf = TarFile.bz2open(tarball_filename)
612.1.1 by Jelmer Vernooij
.tar.lzma -> .tar.xz
814
    elif (tarball_filename.endswith(".tar.lzma") or
815
          tarball_filename.endswith(".tar.xz")):
577.2.1 by Jelmer Vernooij
Support lzma in a couple more places.
816
        tar_args.append('xJf')
1085 by Jelmer Vernooij
Support falling back to strip-components=0 when the orig tarball
817
        tf = TarFile.xzopen(tarball_filename)
818
    elif tarball_filename.endswith(".tar"):
819
        tar_args.append('xf')
820
        tf = TarFile.open(tarball_filename)
570.2.7 by Jelmer Vernooij
Factor out extracting of orig tarballs.
821
    else:
1085 by Jelmer Vernooij
Support falling back to strip-components=0 when the orig tarball
822
        tf = TarFile.gzopen(tarball_filename)
570.2.7 by Jelmer Vernooij
Factor out extracting of orig tarballs.
823
        tar_args.append('xzf')
1085 by Jelmer Vernooij
Support falling back to strip-components=0 when the orig tarball
824
    try:
825
        if strip_components is None:
826
            if needs_strip_components(tf):
827
                strip_components = 1
828
            else:
829
                strip_components = 0
830
    finally:
831
        tf.close()
570.2.16 by Jelmer Vernooij
Extract extract_orig_tarballs
832
    if component is not None:
833
        target_path = os.path.join(target, component)
570.2.17 by Jelmer Vernooij
Pass along component with tarballs.
834
        os.mkdir(target_path)
570.2.16 by Jelmer Vernooij
Extract extract_orig_tarballs
835
    else:
836
        target_path = target
837
    tar_args.extend([tarball_filename, "-C", target_path])
570.2.7 by Jelmer Vernooij
Factor out extracting of orig tarballs.
838
    if strip_components is not None:
1081 by Jelmer Vernooij
Pass through strip-components flag.
839
        tar_args.extend(["--strip-components", str(strip_components)])
1084 by Jelmer Vernooij
Some more reformatting + extra error messages.
840
    proc = subprocess.Popen(tar_args, preexec_fn=subprocess_setup,
841
                            stderr=subprocess.PIPE)
842
    (stdout, stderr) = proc.communicate()
570.2.7 by Jelmer Vernooij
Factor out extracting of orig tarballs.
843
    if proc.returncode != 0:
1084 by Jelmer Vernooij
Some more reformatting + extra error messages.
844
        raise TarFailed("extract", tarball_filename, error=stderr)
570.2.16 by Jelmer Vernooij
Extract extract_orig_tarballs
845
846
847
def extract_orig_tarballs(tarballs, target, strip_components=None):
848
    """Extract orig tarballs to a directory.
849
850
    :param tarballs: List of tarball filenames
851
    :param target: Target directory (must already exist)
852
    """
570.2.17 by Jelmer Vernooij
Pass along component with tarballs.
853
    for tarball_filename, component in tarballs:
1007 by Jelmer Vernooij
Some reformatting.
854
        extract_orig_tarball(
1084 by Jelmer Vernooij
Some more reformatting + extra error messages.
855
            tarball_filename, component, target, strip_components=strip_components)
858.1.4 by Jelmer Vernooij
Move dput_changes to a separate file.
856
857
858
def dput_changes(path):
859
    """Upload a package."""
860
    (bd, changes_file) = os.path.split(path)
861
    subprocess.check_call(["dput", changes_file], cwd=bd)
880 by Jelmer Vernooij
Make changelog_find_previous_upload public.
862
863
938 by Jelmer Vernooij
Add keyid argument to debsign.
864
def debsign(path, keyid=None):
880 by Jelmer Vernooij
Make changelog_find_previous_upload public.
865
    (bd, changes_file) = os.path.split(path)
938 by Jelmer Vernooij
Add keyid argument to debsign.
866
    args = ["debsign"]
867
    if keyid:
868
        args.append("-k%s" % keyid)
869
    args.append(changes_file)
870
    subprocess.check_call(args, cwd=bd)
930 by Jelmer Vernooij
Move do_build and changes_filename.
871
872
873
def changes_filename(package, version, arch):
874
    non_epoch_version = version.upstream_version
875
    if version.debian_version is not None:
876
        non_epoch_version += "-%s" % version.debian_version
1007 by Jelmer Vernooij
Some reformatting.
877
    return "%s_%s_%s.changes" % (package, non_epoch_version, arch)
930 by Jelmer Vernooij
Move do_build and changes_filename.
878
879
933 by Jelmer Vernooij
Move get_build_architecture to utils.
880
def get_build_architecture():
881
    try:
882
        return subprocess.check_output(
883
            ['dpkg-architecture', '-qDEB_BUILD_ARCH']).strip().decode()
884
    except subprocess.CalledProcessError as e:
885
        raise BzrError(
886
            "Could not find the build architecture: %s" % e)
1034 by Jelmer Vernooij
Add function to retrieve Files-Excluded.
887
888
889
def get_files_excluded(tree, subpath='', top_level=False):
890
    if top_level:
891
        path = os.path.join(subpath, 'copyright')
892
    else:
893
        path = os.path.join(subpath, 'debian', 'copyright')
894
    with tree.get_file(path) as f:
1040 by Jelmer Vernooij
Handle NotMachineReadableError.
895
        try:
1043 by Jelmer Vernooij
Don't be strict when parsing copyright.
896
            copyright = Copyright(f, strict=False)
1040 by Jelmer Vernooij
Handle NotMachineReadableError.
897
        except NotMachineReadableError:
898
            return []
1034 by Jelmer Vernooij
Add function to retrieve Files-Excluded.
899
        try:
900
            return copyright.header["Files-Excluded"].split()
901
        except KeyError:
902
            return []