fix ignoreScm

Signed-off-by: Alberto Pianon <alberto@pianon.eu>

aliens4friends/commands/analytics.py

+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Alberto Pianon <pianon@array.eu>
+
+import os
+import re
+import logging
+
+from typing import List
+
+from aliens4friends.commons.settings import Settings
+from aliens4friends.commons.pool import FILETYPE
+from aliens4friends.commands.command import Command, CommandError, Processing
+from aliens4friends.commons.analytics import AnalyticsMetaData
+from aliens4friends.commons.utils import bash
+
+logger = logging.getLogger(__name__)
+
+class Analytics(Command):
+	"""collect and process tinfoilhat data in order to get git repos for all
+	upstream sources, including those not having a 'git' url scheme, in order
+	to provide input (i.e. git repo url, srcrev) to open source project
+	analytics tools"""
+
+	def __init__(self, session_id: str, target_dir: str, dryrun: bool):
+		super().__init__(session_id, Processing.SINGLE, dryrun)
+		self.target_dir = target_dir
+		self.dryrun = dryrun
+
+	def run(self, paths: List[str], target_dir) -> bool:
+		analytics = AnalyticsMetaData(paths, target_dir)
+		analytics.collect()
+		analytics.process()
+		analytics.write()
+		return True
+
+	@staticmethod
+	def execute(
+		target_dir: str,
+		session_id: str = "",
+		dryrun: bool = False
+	) -> bool:
+		if not os.path.isdir(target_dir):
+			bash(f"mkdir -p {target_dir}")
+		cmd = Analytics(session_id, target_dir, dryrun)
+		return cmd.exec_with_paths(FILETYPE.TINFOILHAT)

aliens4friends/commons/analytics.py

+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Alberto Pianon <pianon@array.eu>
+
+import re
+from typing import List
+
+from aliens4friends.models.tinfoilhat import TinfoilHatModel
+
+class AnalyticsMetaData:
+
+	URI_GROUPS = {
+		# name: regex
+		"web": r"^(http|ftp)",
+		"git": r'^git',
+		# TODO: add support for crate, npm, and other uri schemes/bitbake fetchers
+	}
+
+	def __init__(self, tfh_files: List[str], target_dir) -> None:
+		self.tinfoilhat_files = tfh_files
+		self.target_dir = target_dir
+		self.uri_group = {
+			uri_groupname: []
+			for uri_groupname in self.URI_GROUPS
+		}
+
+	def collect(self) -> None:
+		for fname in self.tinfoilhat_files:
+			tfh = TinfoilHatModel.from_file(fname)
+			for k, obj in tfh._container.items():
+				for s in obj.recipe.source_files:
+					for name, regex in self.URI_GROUPS.items():
+						if re.match(regex, s.src_uri):
+							self.uri_group[name].append(s.src_uri.split(";")[0])
+
+	def process(self) -> None:
+		pass
+
+
+# cdn.kernel.org/pub/linux/bluetooth/bluez-5.65.tar.xz
+# cdn.kernel.org/pub/linux/bluetooth/sbc-1.5.tar.xz
+# cdn.kernel.org/pub/linux/kernel/firmware/linux-firmware-20220708.tar.xz'
+# cdn.kernel.org/pub/linux/libs/security/linux-privs/libcap2/libcap-2.65.tar.xz
+# cdn.kernel.org/pub/linux/utils/kbd/kbd-2.4.0.tar.xz
+# cdn.kernel.org/pub/linux/utils/usb/usbutils/usbutils-014.tar.gz
+# cdn.kernel.org/pub/linux/utils/util-linux/v2.37/util-linux-2.37.4.tar.xz
+# cdn.kernel.org/pub/linux/utils/util-linux/v2.37/util-linux-2.37.4.tar.xz
+# cdn.kernel.org/pub/software/utils/pciutils/pciutils-3.7.0.tar.xz
+
+	def write(self) -> None:
+		pass

aliens4friends/commons/fossywrapper.py

@@ -152,7 +152,7 @@ class FossyWrapper:
 			"folderId": str(folder.id),
 			"uploadDescription": description,
 			"uploadType": "file",
-			"ignoreScm": True,
+			"ignoreScm": "true",
 		}
 		data = {
 			"scanOptions": {

current_uris_sample.py

+current_uris_sample = ['https://ftp.gnu.org/gnu/libtool/libtool-2.4.7.tar.gz',
+ 'https://www.alsa-project.org/files/pub/plugins/alsa-plugins-1.2.6.tar.bz2',
+ 'https://sourceware.org/pub/newlib/newlib-4.2.0.20211231.tar.gz',
+ 'https://ftp.gnu.org/gnu/gcc/gcc-11.3.0/gcc-11.3.0.tar.xz',
+ 'https://cdn.kernel.org/pub/linux/libs/security/linux-privs/libcap2/libcap-2.65.tar.xz',
+ 'https://ftp.gnu.org/gnu/gcc/gcc-11.3.0/gcc-11.3.0.tar.xz',
+ 'http://snapshot.debian.org/archive/debian/20160728T043443Z/pool/main/a/apmd/apmd_3.2.2.orig.tar.gz;name=tarball',
+ 'https://www.alsa-project.org/files/pub/lib/alsa-lib-1.2.6.1.tar.bz2',
+ 'https://downloads.sourceforge.net/hdparm/hdparm-9.63.tar.gz',
+ 'http://ftp.mozilla.org/pub/nspr/releases/v4.29/src/nspr-4.29.tar.gz',
+ 'https://github.com/linux-pam/linux-pam/releases/download/v1.5.2/Linux-PAM-1.5.2.tar.xz',
+ 'https://dbus.freedesktop.org/releases/dbus/dbus-1.14.0.tar.xz',
+ 'https://github.com/libfuse/libfuse/releases/download/fuse-3.10.5/fuse-3.10.5.tar.xz',
+ 'http://downloads.xiph.org/releases/flac/flac-1.3.4.tar.xz',
+ 'https://github.com/libsndfile/libsndfile/releases/download/1.0.31/libsndfile-1.0.31.tar.bz2',
+ 'http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz',
+ 'https://sourceware.org/pub/newlib/newlib-4.2.0.20211231.tar.gz',
+ 'https://www.gnupg.org/ftp/gcrypt/libgpg-error/libgpg-error-1.44.tar.bz2',
+ 'https://cdn.kernel.org/pub/software/utils/pciutils/pciutils-3.7.0.tar.xz',
+ 'https://downloads.sourceforge.net/libpng/zlib/1.2.11/zlib-1.2.11.tar.xz',
+ 'https://github.com/libusb/libusb/releases/download/v1.0.26/libusb-1.0.26.tar.bz2',
+ 'http://w1.fi/releases/wpa_supplicant-2.10.tar.gz',
+ 'https://www.alsa-project.org/files/pub/lib/alsa-ucm-conf-1.2.6.3.tar.bz2',
+ 'https://www.alsa-project.org/files/pub/utils/alsa-utils-1.2.6.tar.bz2',
+ 'http://www.sqlite.org/2022/sqlite-autoconf-3380500.tar.gz',
+ 'https://cdn.kernel.org/pub/linux/bluetooth/bluez-5.65.tar.xz',
+ 'https://downloads.sourceforge.net/acpid2/acpid-2.0.33.tar.xz',
+ 'http://ftp.mozilla.org/pub/mozilla.org/security/nss/releases/NSS_3_74_RTM/src/nss-3.74.tar.gz',
+ 'http://0pointer.de/lennart/projects/libdaemon/libdaemon-0.14.tar.gz',
+ 'https://github.com/thom311/libnl/releases/download/libnl3_5_0/libnl-3.5.0.tar.gz',
+ 'https://cdn.kernel.org/pub/linux/utils/kbd/kbd-2.4.0.tar.xz',
+ 'https://github.com/shadow-maint/shadow/releases/download/v4.11.1/shadow-4.11.1.tar.gz',
+ 'https://download.gnome.org/sources//glib/2.72/glib-2.72.3.tar.xz',
+ 'https://www.nxp.com/lgfiles/NMG/MAD/YOCTO//firmware-imx-8.15.bin;fsl-eula=true',
+ 'http://freedesktop.org/software/pulseaudio/releases/pulseaudio-15.0.tar.xz',
+ 'http://netfilter.org/projects/iptables/files/iptables-1.8.7.tar.bz2',
+ 'https://tukaani.org/xz/xz-5.2.6.tar.gz',
+ 'https://download.savannah.gnu.org/releases/attr/attr-2.5.1.tar.gz',
+ 'https://github.com/libffi/libffi/releases/download/v3.4.2/libffi-3.4.2.tar.gz',
+ 'http://ftp.debian.org/debian/pool/main/n/netbase/netbase_6.3.tar.xz',
+ 'http://www.freedesktop.org/software/libmbim/libmbim-1.26.2.tar.xz',
+ 'https://www.gnupg.org/ftp/gcrypt/gpgme/gpgme-1.17.1.tar.bz2',
+ 'https://github.com/lathiat/avahi/releases/download/v0.8/avahi-0.8.tar.gz',
+ 'http://www.kernel.org/pub/software/network/iw/iw-5.16.tar.gz',
+ 'https://busybox.net/downloads/busybox-1.35.0.tar.bz2;name=tarball',
+ 'https://ftp.gnu.org/gnu/libunistring/libunistring-1.0.tar.gz',
+ 'https://github.com/libsndfile/libsamplerate/releases/download/0.2.2/libsamplerate-0.2.2.tar.xz',
+ 'http://downloads.xiph.org/releases/vorbis/libvorbis-1.3.7.tar.xz',
+ 'https://github.com/rauc/rauc-hawkbit-updater/releases/download/v1.1/rauc-hawkbit-updater-1.1.tar.gz',
+ 'https://libbsd.freedesktop.org/releases/libbsd-0.11.5.tar.xz',
+ 'http://downloads.xiph.org/releases/speex/speexdsp-1.2.0.tar.gz',
+ 'https://cdn.kernel.org/pub/linux/utils/util-linux/v2.37/util-linux-2.37.4.tar.xz',
+ 'https://curl.se/download/curl-7.82.0.tar.xz',
+ 'https://ftp.gnu.org/gnu/grub/grub-2.06.tar.gz',
+ 'https://github.com/rauc/rauc/releases/download/v1.7/rauc-1.7.tar.xz',
+ 'https://download.gnome.org/sources//libgudev/237/libgudev-237.tar.xz;name=archive',
+ 'http://www.openssl.org/source/openssl-3.0.5.tar.gz',
+ 'https://download.gnome.org/sources//libxml2/2.9/libxml2-2.9.14.tar.xz;name=archive',
+ 'http://www.w3.org/XML/Test/xmlts20080827.tar.gz;subdir=libxml2-2.9.14;name=testtar',
+ 'https://download.savannah.gnu.org/releases/acl/acl-2.3.1.tar.gz',
+ 'http://snapshot.debian.org/archive/debian/20110427T035506Z/pool/main/v/v86d/v86d_0.1.10.orig.tar.gz',
+ 'http://downloads.xiph.org/releases/ogg/libogg-1.3.5.tar.xz',
+ 'https://www.sudo.ws/dist/sudo-1.9.10.tar.gz',
+ 'https://ftp.gnu.org/gnu/grub/grub-2.06.tar.gz',
+ 'https://ftp.gnu.org/gnu/readline/readline-8.1.2.tar.gz;name=archive',
+ 'https://cdn.kernel.org/pub/linux/utils/util-linux/v2.37/util-linux-2.37.4.tar.xz',
+ 'https://github.com/dosfstools/dosfstools/releases/download/v4.2/dosfstools-4.2.tar.gz',
+ 'https://archive.hadrons.org/software/libmd/libmd-1.0.4.tar.xz',
+ 'https://downloads.sourceforge.net/pcre/pcre-8.45.tar.bz2',
+ 'https://www.kernel.org/pub/software/network/wireless-regdb/wireless-regdb-2022.08.12.tar.xz',
+ 'https://www.gnupg.org/ftp/gcrypt/libassuan/libassuan-2.5.5.tar.bz2',
+ 'https://ftp.gnu.org/gnu/libidn/libidn2-2.3.2.tar.gz',
+ 'http://downloads.sourceforge.net/openobex/obexftp-0.24.2-Source.tar.gz',
+ 'https://download.gnome.org/sources//NetworkManager/1.36/NetworkManager-1.36.2.tar.xz',
+ 'https://www.alsa-project.org/files/pub/lib/alsa-topology-conf-1.2.5.1.tar.bz2',
+ 'https://download.gnome.org/sources//json-glib/1.6/json-glib-1.6.6.tar.xz;name=archive',
+ 'https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz',
+ 'https://cdn.kernel.org/pub/linux/utils/usb/usbutils/usbutils-014.tar.gz',
+ 'http://www.python.org/ftp/python/3.10.4/Python-3.10.4.tar.xz',
+ 'http://matt.ucc.asn.au/dropbear/releases/dropbear-2020.81.tar.bz2',
+ 'https://cdn.kernel.org/pub/linux/bluetooth/sbc-1.5.tar.xz',
+ 'https://github.com/libexpat/libexpat/releases/download/R_2_4_7/expat-2.4.7.tar.bz2',
+ 'http://ftp.openbsd.org/pub/OpenBSD/OpenSSH/portable/openssh-8.9p1.tar.gz',
+ 'https://downloads.sourceforge.net/openobex/openobex/1.7.2/openobex-1.7.2-Source.tar.gz',
+ 'https://cdn.kernel.org/pub/linux/kernel/firmware/linux-firmware-20220708.tar.xz']
+
+
+ curated_uris_sample = [
+  '0pointer.de/lennart/projects/libdaemon/libdaemon-0.14.tar.gz',
+  'archive.hadrons.org/software/libmd/libmd-1.0.4.tar.xz',
+  'busybox.net/downloads/busybox-1.35.0.tar.bz2;name=tarball',
+
+  'curl.se/download/curl-7.82.0.tar.xz',
+  'dbus.freedesktop.org/releases/dbus/dbus-1.14.0.tar.xz',
+  'download.gnome.org/sources//NetworkManager/1.36/NetworkManager-1.36.2.tar.xz',
+  'download.gnome.org/sources//glib/2.72/glib-2.72.3.tar.xz',
+  'download.gnome.org/sources//json-glib/1.6/json-glib-1.6.6.tar.xz;name=archive',
+  'download.gnome.org/sources//libgudev/237/libgudev-237.tar.xz;name=archive',
+  'download.gnome.org/sources//libxml2/2.9/libxml2-2.9.14.tar.xz;name=archive',
+  'download.savannah.gnu.org/releases/acl/acl-2.3.1.tar.gz',
+  'download.savannah.gnu.org/releases/attr/attr-2.5.1.tar.gz',
+  'downloads.sourceforge.net/acpid2/acpid-2.0.33.tar.xz',
+  'downloads.sourceforge.net/hdparm/hdparm-9.63.tar.gz',
+  'downloads.sourceforge.net/libpng/zlib/1.2.11/zlib-1.2.11.tar.xz',
+  'downloads.sourceforge.net/openobex/obexftp-0.24.2-Source.tar.gz',
+  'downloads.sourceforge.net/openobex/openobex/1.7.2/openobex-1.7.2-Source.tar.gz',
+  'downloads.sourceforge.net/pcre/pcre-8.45.tar.bz2',
+  'downloads.xiph.org/releases/flac/flac-1.3.4.tar.xz',
+  'downloads.xiph.org/releases/ogg/libogg-1.3.5.tar.xz',
+  'downloads.xiph.org/releases/speex/speexdsp-1.2.0.tar.gz',
+  'downloads.xiph.org/releases/vorbis/libvorbis-1.3.7.tar.xz',
+  'freedesktop.org/software/pulseaudio/releases/pulseaudio-15.0.tar.xz',
+  'ftp.debian.org/debian/pool/main/n/netbase/netbase_6.3.tar.xz',
+  'ftp.gnu.org/gnu/gcc/gcc-11.3.0/gcc-11.3.0.tar.xz',
+  'ftp.gnu.org/gnu/gcc/gcc-11.3.0/gcc-11.3.0.tar.xz',
+  'ftp.gnu.org/gnu/grub/grub-2.06.tar.gz',
+  'ftp.gnu.org/gnu/grub/grub-2.06.tar.gz',
+  'ftp.gnu.org/gnu/libidn/libidn2-2.3.2.tar.gz',
+  'ftp.gnu.org/gnu/libtool/libtool-2.4.7.tar.gz',
+  'ftp.gnu.org/gnu/libunistring/libunistring-1.0.tar.gz',
+  'ftp.gnu.org/gnu/readline/readline-8.1.2.tar.gz;name=archive',
+  'ftp.mozilla.org/pub/mozilla.org/security/nss/releases/NSS_3_74_RTM/src/nss-3.74.tar.gz',
+  'ftp.mozilla.org/pub/nspr/releases/v4.29/src/nspr-4.29.tar.gz',
+  'ftp.openbsd.org/pub/OpenBSD/OpenSSH/portable/openssh-8.9p1.tar.gz',
+  'github.com/dosfstools/dosfstools/releases/download/v4.2/dosfstools-4.2.tar.gz',
+  'github.com/lathiat/avahi/releases/download/v0.8/avahi-0.8.tar.gz',
+  'github.com/libexpat/libexpat/releases/download/R_2_4_7/expat-2.4.7.tar.bz2',
+  'github.com/libffi/libffi/releases/download/v3.4.2/libffi-3.4.2.tar.gz',
+  'github.com/libfuse/libfuse/releases/download/fuse-3.10.5/fuse-3.10.5.tar.xz',
+  'github.com/libsndfile/libsamplerate/releases/download/0.2.2/libsamplerate-0.2.2.tar.xz',
+  'github.com/libsndfile/libsndfile/releases/download/1.0.31/libsndfile-1.0.31.tar.bz2',
+  'github.com/libusb/libusb/releases/download/v1.0.26/libusb-1.0.26.tar.bz2',
+  'github.com/linux-pam/linux-pam/releases/download/v1.5.2/Linux-PAM-1.5.2.tar.xz',
+  'github.com/rauc/rauc-hawkbit-updater/releases/download/v1.1/rauc-hawkbit-updater-1.1.tar.gz',
+  'github.com/rauc/rauc/releases/download/v1.7/rauc-1.7.tar.xz',
+  'github.com/shadow-maint/shadow/releases/download/v4.11.1/shadow-4.11.1.tar.gz',
+  'github.com/thom311/libnl/releases/download/libnl3_5_0/libnl-3.5.0.tar.gz',
+  'libbsd.freedesktop.org/releases/libbsd-0.11.5.tar.xz',
+  'matt.ucc.asn.au/dropbear/releases/dropbear-2020.81.tar.bz2',
+  'netfilter.org/projects/iptables/files/iptables-1.8.7.tar.bz2',
+  'snapshot.debian.org/archive/debian/20110427T035506Z/pool/main/v/v86d/v86d_0.1.10.orig.tar.gz',
+  'snapshot.debian.org/archive/debian/20160728T043443Z/pool/main/a/apmd/apmd_3.2.2.orig.tar.gz;name=tarball',
+  'sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz',
+  'sourceware.org/pub/newlib/newlib-4.2.0.20211231.tar.gz',
+  'sourceware.org/pub/newlib/newlib-4.2.0.20211231.tar.gz',
+  'tukaani.org/xz/xz-5.2.6.tar.gz',
+  'w1.fi/releases/wpa_supplicant-2.10.tar.gz',
+  'www.alsa-project.org/files/pub/lib/alsa-lib-1.2.6.1.tar.bz2',
+  'www.alsa-project.org/files/pub/lib/alsa-topology-conf-1.2.5.1.tar.bz2',
+  'www.alsa-project.org/files/pub/lib/alsa-ucm-conf-1.2.6.3.tar.bz2',
+  'www.alsa-project.org/files/pub/plugins/alsa-plugins-1.2.6.tar.bz2',
+  'www.alsa-project.org/files/pub/utils/alsa-utils-1.2.6.tar.bz2',
+  'www.freedesktop.org/software/libmbim/libmbim-1.26.2.tar.xz',
+  'www.gnupg.org/ftp/gcrypt/gpgme/gpgme-1.17.1.tar.bz2',
+  'www.gnupg.org/ftp/gcrypt/libassuan/libassuan-2.5.5.tar.bz2',
+  'www.gnupg.org/ftp/gcrypt/libgpg-error/libgpg-error-1.44.tar.bz2',
+  'www.kernel.org/pub/software/network/iw/iw-5.16.tar.gz',
+  'www.kernel.org/pub/software/network/wireless-regdb/wireless-regdb-2022.08.12.tar.xz',
+  'www.nxp.com/lgfiles/NMG/MAD/YOCTO//firmware-imx-8.15.bin;fsl-eula=true',
+  'www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz',
+  'www.openssl.org/source/openssl-3.0.5.tar.gz',
+  'www.python.org/ftp/python/3.10.4/Python-3.10.4.tar.xz',
+  'www.sqlite.org/2022/sqlite-autoconf-3380500.tar.gz',
+  'www.sudo.ws/dist/sudo-1.9.10.tar.gz',
+  'www.w3.org/XML/Test/xmlts20080827.tar.gz;subdir=libxml2-2.9.14;name=testtar',
+ ]