From 8f8b31d9406b216590c4cd74fcc3b21ffa772d8f Mon Sep 17 00:00:00 2001 From: Jelle van der Waa Date: Tue, 8 Oct 2019 23:24:39 +0200 Subject: [PATCH 1/3] devel: add reposec management command Add a new command that scans pkg.tar.xz files with elf binaries in /usr/bin/ and checks for security hardening issues. This adds a new dashboard view which shows packages with these issues. --- devel/management/commands/reposec.py | 236 ++++++++++++++++++++++++ devel/reports.py | 27 ++- main/migrations/0003_packagesecurity.py | 28 +++ main/models.py | 20 ++ requirements.txt | 2 + 5 files changed, 311 insertions(+), 2 deletions(-) create mode 100644 devel/management/commands/reposec.py create mode 100644 main/migrations/0003_packagesecurity.py diff --git a/devel/management/commands/reposec.py b/devel/management/commands/reposec.py new file mode 100644 index 00000000..3db2fb66 --- /dev/null +++ b/devel/management/commands/reposec.py @@ -0,0 +1,236 @@ +""" +reposec command + +Parses all packages in a given repo and creates PackageSecurity +objects which check for PIE, RELRO, Stack Canary's and Fortify. + +Usage: ./manage.py reposec ARCH PATH + ARCH: architecture to check + PATH: full path to the repository directory. + +Example: + ./manage.py reposec x86_64 /srv/ftp/core +""" + +import io +import os +import re +import sys +import logging + +from functools import partial +from glob import glob +from multiprocessing import Pool, cpu_count + +from elftools.elf.constants import P_FLAGS +from elftools.elf.dynamic import DynamicSection +from elftools.elf.elffile import ELFFile +from elftools.elf.sections import SymbolTableSection +from libarchive import file_reader + +from django.core.management.base import BaseCommand, CommandError +from django.db import transaction + +from main.models import Arch, Package, PackageSecurity, Repo + + +PKG_EXT = '.tar.xz' +STACK_CHK = set(["__stack_chk_fail", "__stack_smash_handler"]) + + +logging.basicConfig( + level=logging.WARNING, + format='%(asctime)s -> %(levelname)s: %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + stream=sys.stderr) +TRACE = 5 +logging.addLevelName(TRACE, 'TRACE') +logger = logging.getLogger() + +class Command(BaseCommand): + help = "" + missing_args_message = 'missing arch and file.' + + def add_arguments(self, parser): + parser.add_argument('args', nargs='*', help=' ') + parser.add_argument('--processes', + action='store_true', + dest='processes', + default=cpu_count(), + help=f'number of parallel processes (default: {cpu_count()})') + + + def handle(self, arch=None, directory=None, processes=cpu_count(), **options): + if not arch: + raise CommandError('Architecture is required.') + if not directory: + raise CommandError('Repo location is required.') + directory = os.path.normpath(directory) + if not os.path.exists(directory): + raise CommandError('Specified repository location does not exists.') + + v = int(options.get('verbosity', 0)) + if v == 0: + logger.level = logging.ERROR + elif v == 1: + logger.level = logging.INFO + elif v >= 2: + logger.level = logging.DEBUG + + return read_repo(arch, directory, processes, options) + + +def read_file(arch, repo, filename): + pkgsec = None + basename = os.path.basename(filename) + pkgname = basename.rsplit('-', 3)[0] + + with file_reader(filename) as pkg: + for entry in pkg: + if not entry.isfile: + continue + + # Retrieve pkgname + if entry.name == '.PKGINFO': + continue + + if not entry.name.startswith('usr/bin/'): + continue + + fp = io.BytesIO(b''.join(entry.get_blocks())) + elf = Elf(fp) + + if not elf.is_elf(): + continue + + pkg = Package.objects.get(pkgname=pkgname, arch=arch, repo=repo) + pkgsec = PackageSecurity(pkg=pkg, pie=elf.pie, relro=elf.relro, canary=elf.canary) + + return pkgsec + + + +def read_repo(arch, source_dir, processes, options): + tasks = [] + + directory = os.path.join(source_dir, 'os', arch) + for filename in glob(os.path.join(directory, f'*{PKG_EXT}')): + tasks.append((filename)) + + arch = Arch.objects.get(name=arch) + + reponame = os.path.basename(source_dir).title() + repo = Repo.objects.get(name=reponame) + + + with Pool(processes=processes) as pool: + results = pool.map(partial(read_file, arch, repo), tasks) + + results = [r for r in results if r and (not r.pie or not r.relro or not r.canary)] + + with transaction.atomic(): + PackageSecurity.objects.all().delete() + PackageSecurity.objects.bulk_create(results) + + +class Elf: + def __init__(self, fileobj): + self.fileobj = fileobj + self._elffile = None + + @property + def elffile(self): + if not self._elffile: + self._elffile = ELFFile(self.fileobj) + return self._elffile + + def _file_has_magic(self, fileobj, magic_bytes): + length = len(magic_bytes) + magic = fileobj.read(length) + fileobj.seek(0) + return magic == magic_bytes + + def is_elf(self): + "Take file object, peek at the magic bytes to check if ELF file." + return self._file_has_magic(self.fileobj, b"\x7fELF") + + def dynamic_tags(self, key): + for section in self.elffile.iter_sections(): + if not isinstance(section, DynamicSection): + continue + for tag in section.iter_tags(): + if tag.entry.d_tag == key: + return tag + return None + + def rpath(self, key="DT_RPATH", verbose=False): + tag = self.dynamic_tags(key) + if tag and verbose: + return tag.rpath + if tag: + return 'RPATH' + return '' + + def runpath(self, key="DT_RUNPATH", verbose=False): + tag = self.dynamic_tags(key) + if tag and verbose: + return tag.runpath + if tag: + return 'RUNPATH' + + return '' + + @property + def relro(self): + if self.elffile.num_segments() == 0: + return "Disabled" + + have_relro = False + for segment in self.elffile.iter_segments(): + if re.search("GNU_RELRO", str(segment['p_type'])): + have_relro = True + break + + if self.dynamic_tags("DT_BIND_NOW") and have_relro: + return True + if have_relro: # partial + return False + return False + + @property + def pie(self): + header = self.elffile.header + if self.dynamic_tags("EXEC"): + return "Disabled" + if "ET_DYN" in header['e_type']: + if self.dynamic_tags("DT_DEBUG"): + return True + return True # DSO is PIE + return False + + @property + def canary(self): + for section in self.elffile.iter_sections(): + if not isinstance(section, SymbolTableSection): + continue + if section['sh_entsize'] == 0: + continue + for _, symbol in enumerate(section.iter_symbols()): + if symbol.name in STACK_CHK: + return True + return False + + def program_headers(self): + pflags = P_FLAGS() + if self.elffile.num_segments() == 0: + return "" + + found = False + for segment in self.elffile.iter_segments(): + if search("GNU_STACK", str(segment['p_type'])): + found = True + if segment['p_flags'] & pflags.PF_X: + return "Disabled" + if found: + return "Enabled" + return "Disabled" diff --git a/devel/reports.py b/devel/reports.py index b4f9d794..cab8f9ff 100644 --- a/devel/reports.py +++ b/devel/reports.py @@ -5,7 +5,7 @@ from django.template.defaultfilters import filesizeformat from django.db import connection from django.utils.timezone import now -from main.models import Package, PackageFile +from main.models import Package, PackageFile, PackageSecurity from packages.models import Depend, PackageRelation from .models import DeveloperKey @@ -167,6 +167,20 @@ def non_existing_dependencies(packages): return packages +def security_packages_overview(packages): + filtered = [] + packages_ids = packages.values_list('id', + flat=True).order_by().distinct() + packages = PackageSecurity.objects.filter(id__in=set(packages_ids)) + for package in packages: + package.pkg.pie = 'PIE enabled' if package.pie else 'No PIE' + package.pkg.relro = 'Full RELRO' if package.relro else 'None' + package.pkg.canary = 'Canary found' if package.canary else 'No canary found' + package.pkg.fortify = 'Yes' if package.canary else 'No' + filtered.append(package.pkg) + + return filtered + REPORT_OLD = DeveloperReport( 'old', 'Old', 'Packages last built more than two years ago', old) @@ -223,6 +237,14 @@ def non_existing_dependencies(packages): ['nonexistingdep'], personal=False) +REPORT_SECURITY = DeveloperReport( + 'security-issue-packages', + 'Security of Packages', + 'Packages that have security issues', + security_packages_overview, + ['PIE', 'RELRO', 'CANARY', 'FORTIFY'], ['pie', 'relro', 'canary', 'fortify']) + + def available_reports(): return (REPORT_OLD, REPORT_OUTOFDATE, @@ -233,4 +255,5 @@ def available_reports(): REPORT_ORPHANS, REPORT_SIGNATURE, REPORT_SIG_TIME, - NON_EXISTING_DEPENDENCIES, ) + NON_EXISTING_DEPENDENCIES, + REPORT_SECURITY, ) diff --git a/main/migrations/0003_packagesecurity.py b/main/migrations/0003_packagesecurity.py new file mode 100644 index 00000000..6b1ca416 --- /dev/null +++ b/main/migrations/0003_packagesecurity.py @@ -0,0 +1,28 @@ +# Generated by Django 2.2.5 on 2019-10-09 19:24 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0002_repo_public_testing'), + ] + + operations = [ + migrations.CreateModel( + name='PackageSecurity', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('pie', models.BooleanField(default=False)), + ('relro', models.PositiveIntegerField(choices=[(1, 'No RELRO'), (2, 'Partial RELRO'), (2, 'Full RELRO')], default=1)), + ('canary', models.BooleanField(default=False)), + ('fortify', models.BooleanField(default=False)), + ('pkg', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='main.Package')), + ], + options={ + 'db_table': 'package_security', + }, + ), + ] diff --git a/main/models.py b/main/models.py index 094e80a9..2ef4a1af 100644 --- a/main/models.py +++ b/main/models.py @@ -444,6 +444,26 @@ class Meta: db_table = 'package_files' +class PackageSecurity(models.Model): + NO_RELRO = 1 + PARTIAL_RELRO = 2 + FULL_RELRO = 2 + RELRO_CHOICES = ( + (NO_RELRO, 'No RELRO'), + (PARTIAL_RELRO, 'Partial RELRO'), + (FULL_RELRO, 'Full RELRO'), + ) + + pkg = models.ForeignKey(Package, on_delete=models.CASCADE) + pie = models.BooleanField(default=False) + relro = models.PositiveIntegerField(choices=RELRO_CHOICES, default=NO_RELRO) + canary = models.BooleanField(default=False) + fortify = models.BooleanField(default=False) + + class Meta: + db_table = 'package_security' + + from django.db.models.signals import pre_save # note: reporead sets the 'created' field on Package objects, so no signal diff --git a/requirements.txt b/requirements.txt index 841916b6..e0982db3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,5 @@ django-jinja==2.4.1 sqlparse==0.3.0 django-csp==3.5 ptpython==2.0.4 +pyelftools==0.25 +libarchive-c==2.8 From 0b098b4372c228c4a5a1c6b703d365737b33f06b Mon Sep 17 00:00:00 2001 From: Jelle van der Waa Date: Thu, 10 Oct 2019 23:21:46 +0200 Subject: [PATCH 2/3] wip --- devel/management/commands/reposec.py | 38 ++++++++++++++++++---------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/devel/management/commands/reposec.py b/devel/management/commands/reposec.py index 3db2fb66..33748296 100644 --- a/devel/management/commands/reposec.py +++ b/devel/management/commands/reposec.py @@ -80,20 +80,21 @@ def handle(self, arch=None, directory=None, processes=cpu_count(), **options): return read_repo(arch, directory, processes, options) -def read_file(arch, repo, filename): - pkgsec = None - basename = os.path.basename(filename) - pkgname = basename.rsplit('-', 3)[0] - +def read_file(filename): + ''' + Read a pacman package and determine the 'package security status' by + finding all ELF files and determining the worst status of all ELF files in + the repo. + ''' + + pkgsec = {} + elffiles = [] + with file_reader(filename) as pkg: for entry in pkg: if not entry.isfile: continue - # Retrieve pkgname - if entry.name == '.PKGINFO': - continue - if not entry.name.startswith('usr/bin/'): continue @@ -103,10 +104,14 @@ def read_file(arch, repo, filename): if not elf.is_elf(): continue - pkg = Package.objects.get(pkgname=pkgname, arch=arch, repo=repo) - pkgsec = PackageSecurity(pkg=pkg, pie=elf.pie, relro=elf.relro, canary=elf.canary) + elffiles.append(elf) - return pkgsec + + if elffiles: + elf = elffiles[0] + print(elffiles) + + yield pkgsec @@ -124,9 +129,14 @@ def read_repo(arch, source_dir, processes, options): with Pool(processes=processes) as pool: - results = pool.map(partial(read_file, arch, repo), tasks) + results = pool.imap(read_file, tasks) + + # Process and add Package object. + results = [r for r in results if r] + resutls = [r for r in results if not r.pie or not r.relro or not r.canary] - results = [r for r in results if r and (not r.pie or not r.relro or not r.canary)] + print(results) + return with transaction.atomic(): PackageSecurity.objects.all().delete() From c8f731745d3a0fbd0ab31e34e9a067e3a49cc63b Mon Sep 17 00:00:00 2001 From: Jelle van der Waa Date: Sat, 12 Oct 2019 23:31:31 +0200 Subject: [PATCH 3/3] rewrite logic to show all files with their status Show all offending files with security hardening bits missing --- devel/management/commands/reposec.py | 114 ++++++++++-------- devel/reports.py | 7 +- .../0004_packagesecurity_filename.py | 18 +++ main/models.py | 9 +- 4 files changed, 92 insertions(+), 56 deletions(-) create mode 100644 main/migrations/0004_packagesecurity_filename.py diff --git a/devel/management/commands/reposec.py b/devel/management/commands/reposec.py index 33748296..2c7d617d 100644 --- a/devel/management/commands/reposec.py +++ b/devel/management/commands/reposec.py @@ -18,7 +18,6 @@ import sys import logging -from functools import partial from glob import glob from multiprocessing import Pool, cpu_count @@ -34,7 +33,8 @@ from main.models import Arch, Package, PackageSecurity, Repo -PKG_EXT = '.tar.xz' +PKG_EXT = '.tar.xz' # TODO: detect zstd.. +BIN_PATHS = ['usr/bin/', 'opt/'] STACK_CHK = set(["__stack_chk_fail", "__stack_smash_handler"]) @@ -48,7 +48,7 @@ logger = logging.getLogger() class Command(BaseCommand): - help = "" + help = "Checks all packages in the repository for missing hardening bits (relro, stack canary, pie, etc)" missing_args_message = 'missing arch and file.' def add_arguments(self, parser): @@ -87,32 +87,32 @@ def read_file(filename): the repo. ''' - pkgsec = {} elffiles = [] with file_reader(filename) as pkg: + pkgname = os.path.basename(filename).rsplit('-', 3)[0] + for entry in pkg: if not entry.isfile: continue - if not entry.name.startswith('usr/bin/'): + if not any(entry.name.startswith(path) for path in BIN_PATHS): continue fp = io.BytesIO(b''.join(entry.get_blocks())) - elf = Elf(fp) + elf = Elf(entry.name, fp) if not elf.is_elf(): continue - elffiles.append(elf) + if elf.hardened: + continue - - if elffiles: - elf = elffiles[0] - print(elffiles) - - yield pkgsec + data = elf.dump() + data['pkgname'] = pkgname + elffiles.append(data) + return elffiles def read_repo(arch, source_dir, processes, options): @@ -127,24 +127,37 @@ def read_repo(arch, source_dir, processes, options): reponame = os.path.basename(source_dir).title() repo = Repo.objects.get(name=reponame) + packagesecs = [] with Pool(processes=processes) as pool: - results = pool.imap(read_file, tasks) - - # Process and add Package object. - results = [r for r in results if r] - resutls = [r for r in results if not r.pie or not r.relro or not r.canary] - - print(results) - return + for results in pool.imap_unordered(read_file, tasks): + # No elf files + if not results: + continue + # determine + print(results) + for result in results: + try: + pkg = Package.objects.get(pkgname=result['pkgname'], arch=arch, repo=repo) + except Exception: + print("package '%s' not found in repo" % result['pkgname']) + continue + + print(result) + packagesec = PackageSecurity(pkg=pkg, relro=result['relro'], + pie=result['pie'], canary=result['canary'], + filename=result['filename']) + packagesecs.append(packagesec) + + print(packagesecs) with transaction.atomic(): - PackageSecurity.objects.all().delete() - PackageSecurity.objects.bulk_create(results) + PackageSecurity.objects.bulk_create(packagesecs) class Elf: - def __init__(self, fileobj): + def __init__(self, filename, fileobj): + self.filename = filename self.fileobj = fileobj self._elffile = None @@ -154,16 +167,14 @@ def elffile(self): self._elffile = ELFFile(self.fileobj) return self._elffile - def _file_has_magic(self, fileobj, magic_bytes): + def is_elf(self): + "Take file object, peek at the magic bytes to check if ELF file." + magic_bytes = b"\x7fELF" length = len(magic_bytes) - magic = fileobj.read(length) - fileobj.seek(0) + magic = self.fileobj.read(length) + self.fileobj.seek(0) return magic == magic_bytes - def is_elf(self): - "Take file object, peek at the magic bytes to check if ELF file." - return self._file_has_magic(self.fileobj, b"\x7fELF") - def dynamic_tags(self, key): for section in self.elffile.iter_sections(): if not isinstance(section, DynamicSection): @@ -173,6 +184,7 @@ def dynamic_tags(self, key): return tag return None + @property def rpath(self, key="DT_RPATH", verbose=False): tag = self.dynamic_tags(key) if tag and verbose: @@ -181,6 +193,7 @@ def rpath(self, key="DT_RPATH", verbose=False): return 'RPATH' return '' + @property def runpath(self, key="DT_RUNPATH", verbose=False): tag = self.dynamic_tags(key) if tag and verbose: @@ -193,25 +206,25 @@ def runpath(self, key="DT_RUNPATH", verbose=False): @property def relro(self): if self.elffile.num_segments() == 0: - return "Disabled" + return PackageSecurity.NO_RELRO - have_relro = False + have_relro = PackageSecurity.NO_RELRO for segment in self.elffile.iter_segments(): if re.search("GNU_RELRO", str(segment['p_type'])): - have_relro = True + have_relro = PackageSecurity.FULL_RELRO break if self.dynamic_tags("DT_BIND_NOW") and have_relro: - return True + return PackageSecurity.FULL_RELRO if have_relro: # partial - return False - return False + return PackageSecurity.PARTIAL_RELRO + return PackageSecurity.FULL_RELRO @property def pie(self): header = self.elffile.header if self.dynamic_tags("EXEC"): - return "Disabled" + return False if "ET_DYN" in header['e_type']: if self.dynamic_tags("DT_DEBUG"): return True @@ -230,17 +243,14 @@ def canary(self): return True return False - def program_headers(self): - pflags = P_FLAGS() - if self.elffile.num_segments() == 0: - return "" - - found = False - for segment in self.elffile.iter_segments(): - if search("GNU_STACK", str(segment['p_type'])): - found = True - if segment['p_flags'] & pflags.PF_X: - return "Disabled" - if found: - return "Enabled" - return "Disabled" + @property + def hardened(self): + return self.pie and self.canary and self.relro == PackageSecurity.FULL_RELRO + + def dump(self): + return { + 'pie': self.pie, + 'relro': self.relro, + 'canary': self.canary, + 'filename': self.filename + } diff --git a/devel/reports.py b/devel/reports.py index cab8f9ff..fa10ae0e 100644 --- a/devel/reports.py +++ b/devel/reports.py @@ -171,12 +171,13 @@ def security_packages_overview(packages): filtered = [] packages_ids = packages.values_list('id', flat=True).order_by().distinct() - packages = PackageSecurity.objects.filter(id__in=set(packages_ids)) + packages = PackageSecurity.objects.filter(pkg_id__in=set(packages_ids)) for package in packages: package.pkg.pie = 'PIE enabled' if package.pie else 'No PIE' - package.pkg.relro = 'Full RELRO' if package.relro else 'None' + package.pkg.relro = package.relro_str() package.pkg.canary = 'Canary found' if package.canary else 'No canary found' package.pkg.fortify = 'Yes' if package.canary else 'No' + package.pkg.filename = package.filename filtered.append(package.pkg) return filtered @@ -242,7 +243,7 @@ def security_packages_overview(packages): 'Security of Packages', 'Packages that have security issues', security_packages_overview, - ['PIE', 'RELRO', 'CANARY', 'FORTIFY'], ['pie', 'relro', 'canary', 'fortify']) + ['filename', 'PIE', 'RELRO', 'CANARY', 'FORTIFY'], ['filename', 'pie', 'relro', 'canary', 'fortify']) def available_reports(): diff --git a/main/migrations/0004_packagesecurity_filename.py b/main/migrations/0004_packagesecurity_filename.py new file mode 100644 index 00000000..e0a79cda --- /dev/null +++ b/main/migrations/0004_packagesecurity_filename.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2019-10-12 19:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0003_packagesecurity'), + ] + + operations = [ + migrations.AddField( + model_name='packagesecurity', + name='filename', + field=models.CharField(default='', max_length=1024), + ), + ] diff --git a/main/models.py b/main/models.py index 2ef4a1af..39d8e609 100644 --- a/main/models.py +++ b/main/models.py @@ -447,7 +447,7 @@ class Meta: class PackageSecurity(models.Model): NO_RELRO = 1 PARTIAL_RELRO = 2 - FULL_RELRO = 2 + FULL_RELRO = 3 RELRO_CHOICES = ( (NO_RELRO, 'No RELRO'), (PARTIAL_RELRO, 'Partial RELRO'), @@ -459,6 +459,13 @@ class PackageSecurity(models.Model): relro = models.PositiveIntegerField(choices=RELRO_CHOICES, default=NO_RELRO) canary = models.BooleanField(default=False) fortify = models.BooleanField(default=False) + filename = models.CharField(max_length=1024, blank=False, default='') + + def relro_str(self): + for id_, desc in self.RELRO_CHOICES: + if id_ == self.relro: + return desc + return '' class Meta: db_table = 'package_security'