[PATCH 1/3] copied android.mk from devel branch

[PATCH 1/3] copied android.mk from devel branch

[filipbystricky]

From: Filip Bystricky <filipbystricky@google.com>

This series of patches fixes some compile errors that trigger when
compiling to android devices.

This first patch just brings in devel's Android.mk, to which
kdave@ added a few fixes recently.

Signed-off-by: Filip Bystricky <filipbystricky@google.com>
Reviewed-by: Mark Salyzyn <salyzyn@android.com>
---
 Android.mk | 28 +++++++++++++++++-----------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/Android.mk b/Android.mk
index fe3209b6..52fe9ab4 100644
--- a/Android.mk
+++ b/Android.mk
@@ -17,22 +17,27 @@ STATIC_LIBS := -luuid   -lblkid -luuid -lz   -llzo2 -L. -pthread
 btrfs_shared_libraries := libext2_uuid \
 			libext2_blkid
 
-objects := ctree.c disk-io.c radix-tree.c extent-tree.c print-tree.c \
+objects := ctree.c disk-io.c kernel-lib/radix-tree.c extent-tree.c print-tree.c \
           root-tree.c dir-item.c file-item.c inode-item.c inode-map.c \
           extent-cache.c extent_io.c volumes.c utils.c repair.c \
-          qgroup.c raid6.c free-space-cache.c list_sort.c props.c \
-          ulist.c qgroup-verify.c backref.c string-table.c task-utils.c \
-          inode.c file.c find-root.c
+          qgroup.c free-space-cache.c kernel-lib/list_sort.c props.c \
+          kernel-shared/ulist.c qgroup-verify.c backref.c string-table.c task-utils.c \
+          inode.c file.c find-root.c free-space-tree.c help.c send-dump.c \
+          fsfeatures.c kernel-lib/tables.c kernel-lib/raid56.c
 cmds_objects := cmds-subvolume.c cmds-filesystem.c cmds-device.c cmds-scrub.c \
                cmds-inspect.c cmds-balance.c cmds-send.c cmds-receive.c \
                cmds-quota.c cmds-qgroup.c cmds-replace.c cmds-check.c \
                cmds-restore.c cmds-rescue.c chunk-recover.c super-recover.c \
-               cmds-property.c cmds-fi-usage.c
-libbtrfs_objects := send-stream.c send-utils.c rbtree.c btrfs-list.c crc32c.c \
+               cmds-property.c cmds-fi-usage.c cmds-inspect-dump-tree.c \
+               cmds-inspect-dump-super.c cmds-inspect-tree-stats.c cmds-fi-du.c \
+               mkfs/common.c
+libbtrfs_objects := send-stream.c send-utils.c kernel-lib/rbtree.c btrfs-list.c \
+                   kernel-lib/crc32c.c messages.c \
                    uuid-tree.c utils-lib.c rbtree-utils.c
-libbtrfs_headers := send-stream.h send-utils.h send.h rbtree.h btrfs-list.h \
-                   crc32c.h list.h kerncompat.h radix-tree.h extent-cache.h \
-                   extent_io.h ioctl.h ctree.h btrfsck.h version.h
+libbtrfs_headers := send-stream.h send-utils.h send.h kernel-lib/rbtree.h btrfs-list.h \
+                   kernel-lib/crc32c.h kernel-lib/list.h kerncompat.h \
+                   kernel-lib/radix-tree.h kernel-lib/sizes.h kernel-lib/raid56.h \
+                   extent-cache.h extent_io.h ioctl.h ctree.h btrfsck.h version.h
 TESTS := fsck-tests.sh convert-tests.sh
 blkid_objects := partition/ superblocks/ topology/
 
@@ -75,7 +80,8 @@ include $(CLEAR_VARS)
 LOCAL_MODULE := mkfs.btrfs
 LOCAL_SRC_FILES := \
                 $(objects) \
-                mkfs.c
+                mkfs/common.c \
+                mkfs/main.c
 
 LOCAL_C_INCLUDES := $(common_C_INCLUDES)
 LOCAL_CFLAGS := $(STATIC_CFLAGS)
@@ -108,4 +114,4 @@ LOCAL_SYSTEM_SHARED_LIBRARIES := libc libcutils
 LOCAL_EXPORT_C_INCLUDES := $(common_C_INCLUDES)
 LOCAL_MODULE_TAGS := optional
 include $(BUILD_EXECUTABLE)
-#--------------------------------------------------------------
+#--------------------------------------------------------------
\ No newline at end of file
-- 
2.14.0.rc1.383.gd1ce394fe2-goog

[PATCH 2/3] fixed android.mk

[filipbystricky]

From: Filip Bystricky <filipbystricky@google.com>

Signed-off-by: Filip Bystricky <filipbystricky@google.com>
Reviewed-by: Mark Salyzyn <salyzyn@android.com>
---
 Android.mk | 53 +++++++++++++++++++++--------------------------------
 1 file changed, 21 insertions(+), 32 deletions(-)

diff --git a/Android.mk b/Android.mk
index 52fe9ab4..9516c2d1 100644
--- a/Android.mk
+++ b/Android.mk
@@ -1,18 +1,19 @@
 LOCAL_PATH:= $(call my-dir)
 
-#include $(call all-subdir-makefiles)
+# temporary flags to reduce the number of emitted warnings until they can be
+# fixed properly
+TEMP_CFLAGS := -Wno-pointer-arith -Wno-tautological-constant-out-of-range-compare \
+	-Wno-sign-compare -Wno-format -Wno-unused-parameter
 
 CFLAGS := -g -O1 -Wall -D_FORTIFY_SOURCE=2 -include config.h \
-	-DBTRFS_FLAT_INCLUDES -D_XOPEN_SOURCE=700 -fno-strict-aliasing -fPIC
+	-DBTRFS_FLAT_INCLUDES -D_XOPEN_SOURCE=700 -fno-strict-aliasing -fPIC \
+	-Wno-macro-redefined -Wno-typedef-redefinition -Wno-address-of-packed-member \
+	-Wno-missing-field-initializers $(TEMP_CFLAGS)
 
-LDFLAGS := -static -rdynamic
-
-LIBS := -luuid   -lblkid   -lz   -llzo2 -L. -lpthread
-LIBBTRFS_LIBS := $(LIBS)
-
-STATIC_CFLAGS := $(CFLAGS) -ffunction-sections -fdata-sections
-STATIC_LDFLAGS := -static -Wl,--gc-sections
-STATIC_LIBS := -luuid   -lblkid -luuid -lz   -llzo2 -L. -pthread
+STATIC_CFLAGS := $(CFLAGS) -ffunction-sections -fdata-sections \
+	-D_GNU_SOURCE=1 \
+	-DPACKAGE_STRING=\"btrfs\" \
+	-DPACKAGE_URL=\"http://btrfs.wiki.kernel.org\"
 
 btrfs_shared_libraries := libext2_uuid \
 			libext2_blkid
@@ -23,7 +24,8 @@ objects := ctree.c disk-io.c kernel-lib/radix-tree.c extent-tree.c print-tree.c
           qgroup.c free-space-cache.c kernel-lib/list_sort.c props.c \
           kernel-shared/ulist.c qgroup-verify.c backref.c string-table.c task-utils.c \
           inode.c file.c find-root.c free-space-tree.c help.c send-dump.c \
-          fsfeatures.c kernel-lib/tables.c kernel-lib/raid56.c
+          fsfeatures.c raid56.c
+
 cmds_objects := cmds-subvolume.c cmds-filesystem.c cmds-device.c cmds-scrub.c \
                cmds-inspect.c cmds-balance.c cmds-send.c cmds-receive.c \
                cmds-quota.c cmds-qgroup.c cmds-replace.c cmds-check.c \
@@ -38,12 +40,11 @@ libbtrfs_headers := send-stream.h send-utils.h send.h kernel-lib/rbtree.h btrfs-
                    kernel-lib/crc32c.h kernel-lib/list.h kerncompat.h \
                    kernel-lib/radix-tree.h kernel-lib/sizes.h kernel-lib/raid56.h \
                    extent-cache.h extent_io.h ioctl.h ctree.h btrfsck.h version.h
-TESTS := fsck-tests.sh convert-tests.sh
-blkid_objects := partition/ superblocks/ topology/
-
 
 # external/e2fsprogs/lib is needed for uuid/uuid.h
-common_C_INCLUDES := $(LOCAL_PATH) external/e2fsprogs/lib/ external/lzo/include/ external/zlib/
+common_C_INCLUDES := $(LOCAL_PATH) external/e2fsprogs/lib/ external/lzo/include/ external/zlib/ \
+		$(LOCAL_PATH)/kernel-lib
+
 
 #----------------------------------------------------------
 include $(CLEAR_VARS)
@@ -56,23 +57,18 @@ include $(BUILD_STATIC_LIBRARY)
 #----------------------------------------------------------
 include $(CLEAR_VARS)
 LOCAL_MODULE := btrfs
-#LOCAL_FORCE_STATIC_EXECUTABLE := true
 LOCAL_SRC_FILES := \
 		$(objects) \
 		$(cmds_objects) \
-		btrfs.c \
-		help.c \
+		btrfs.c
 
 LOCAL_C_INCLUDES := $(common_C_INCLUDES)
 LOCAL_CFLAGS := $(STATIC_CFLAGS)
-#LOCAL_LDLIBS := $(LIBBTRFS_LIBS)
-#LOCAL_LDFLAGS := $(STATIC_LDFLAGS)
 LOCAL_SHARED_LIBRARIES := $(btrfs_shared_libraries)
 LOCAL_STATIC_LIBRARIES := libbtrfs liblzo-static libz
 LOCAL_SYSTEM_SHARED_LIBRARIES := libc libcutils
-
 LOCAL_EXPORT_C_INCLUDES := $(common_C_INCLUDES)
-#LOCAL_MODULE_TAGS := optional
+
 include $(BUILD_EXECUTABLE)
 
 #----------------------------------------------------------
@@ -85,14 +81,11 @@ LOCAL_SRC_FILES := \
 
 LOCAL_C_INCLUDES := $(common_C_INCLUDES)
 LOCAL_CFLAGS := $(STATIC_CFLAGS)
-#LOCAL_LDLIBS := $(LIBBTRFS_LIBS)
-#LOCAL_LDFLAGS := $(STATIC_LDFLAGS)
 LOCAL_SHARED_LIBRARIES := $(btrfs_shared_libraries)
 LOCAL_STATIC_LIBRARIES := libbtrfs liblzo-static
 LOCAL_SYSTEM_SHARED_LIBRARIES := libc libcutils
-
 LOCAL_EXPORT_C_INCLUDES := $(common_C_INCLUDES)
-#LOCAL_MODULE_TAGS := optional
+
 include $(BUILD_EXECUTABLE)
 
 #---------------------------------------------------------------
@@ -105,13 +98,9 @@ LOCAL_SRC_FILES := \
 LOCAL_C_INCLUDES := $(common_C_INCLUDES)
 LOCAL_CFLAGS := $(STATIC_CFLAGS)
 LOCAL_SHARED_LIBRARIES := $(btrfs_shared_libraries)
-#LOCAL_LDLIBS := $(LIBBTRFS_LIBS)
-#LOCAL_LDFLAGS := $(STATIC_LDFLAGS)
-LOCAL_SHARED_LIBRARIES := $(btrfs_shared_libraries)
 LOCAL_STATIC_LIBRARIES := libbtrfs liblzo-static
 LOCAL_SYSTEM_SHARED_LIBRARIES := libc libcutils
-
 LOCAL_EXPORT_C_INCLUDES := $(common_C_INCLUDES)
-LOCAL_MODULE_TAGS := optional
+
 include $(BUILD_EXECUTABLE)
-#--------------------------------------------------------------
\ No newline at end of file
+#--------------------------------------------------------------
-- 
2.14.0.rc1.383.gd1ce394fe2-goog

[PATCH 3/3] compile error fixes

[filipbystricky]

From: Filip Bystricky <filipbystricky@google.com>

Android currently does not fully support libblkid, and android's bionic 
doesn't implement some pthread extras such as pthread_tryjoin_np and 
pthread_cancel. This patch fixes the resulting errors while trying to 
be as unobtrusive as possible, and is therefore just a temporary fix. 
For complete support of tools that use background tasks, the way those
are managed (in particular, how they are cancelled) would need to be 
reworked.

Signed-off-by: Filip Bystricky <filipbystricky@google.com>
Reviewed-by: Mark Salyzyn <salyzyn@android.com>
---
 androidcompat.h | 38 ++++++++++++++++++++++++++++++++------
 cmds-scrub.c    |  5 +++++
 mkfs/common.c   |  8 ++++++++
 mkfs/main.c     |  7 +++++++
 task-utils.c    |  1 +
 utils.c         | 18 ++++++++++++++++++
 utils.h         |  1 +
 7 files changed, 72 insertions(+), 6 deletions(-)

diff --git a/androidcompat.h b/androidcompat.h
index eec76dad..bd0be172 100644
--- a/androidcompat.h
+++ b/androidcompat.h
@@ -7,22 +7,48 @@
 #ifndef __ANDROID_H__
 #define __ANDROID_H__
 
-#ifdef ANDROID
-
-#define pthread_setcanceltype(type, oldtype)	(0)
-#define pthread_setcancelstate(state, oldstate)	(0)
+#ifdef __BIONIC__
 
+/*
+ * Bionic doesn't implement pthread_cancel or helpers.
+ *
+ * TODO: this is a temporary fix to just get the tools to compile.
+ * What we really want is to rework how background tasks are managed.
+ * All of the threads that are being cancelled are running in infinite loops.
+ * They should instead be checking a flag at each iteration to see if they
+ * should continue. Then cancelling would just be a matter of setting the flag.
+ *
+ * Most background tasks are managed using btrfs's task_utils library, in which
+ * case they are passed a task_ctx struct pointer.
+ *
+ * However, in two cases, they are created and cancelled directly with the pthread library:
+ *   - chunk-recover.c:scan_devices creates a thread for each device to scan, giving
+ *     each a struct device_scan*.
+ *   - cmds-scrub.c:scrub_start creates a single thread and gives it a struct task_ctx*.
+ *
+ * Breakdown by command:
+ *   - btrfs check (cmds-check.c) uses a task (task_ctx) for indicating progress
+ *   - mkfs.btrfs (mkfs/main.c) doesn't appear to use any background tasks.
+ */
 #define pthread_cancel(ret)	pthread_kill((ret), SIGUSR1)
 
+/*
+ * If given pointers are non-null, just zero out the pointed-to value.
+ * This also eliminates some unused variable warnings.
+ */
+#define pthread_setcanceltype(type, oldtype)	((oldtype) ? (*(oldtype) = 0) : 0)
+#define pthread_setcancelstate(state, oldstate)	((oldstate) ? (*(oldstate) = 0) : 0)
+#define pthread_tryjoin_np(thread, retval) ((retval) ? ((int)(*(retval) = NULL)) : 0)
+
 typedef struct blkid_struct_probe *blkid_probe;
 
 #include <dirent.h>
 #define direct dirent
 
-#else	/* !ANDROID */
+#else	/* !__BIONIC__ */
 
 #include <sys/dir.h>
 
-#endif	/* !ANDROID */
+#endif	/* !__BIONIC__ */
 
 #endif	/* __ANDROID_H__ */
diff --git a/cmds-scrub.c b/cmds-scrub.c
index 5388fdcf..5d8f6c24 100644
--- a/cmds-scrub.c
+++ b/cmds-scrub.c
@@ -46,6 +46,11 @@
 #include "commands.h"
 #include "help.h"
 
+#if defined(__BIONIC__) && !defined(PTHREAD_CANCELED)
+/* bionic's pthread does not define PTHREAD_CANCELED */
+#define PTHREAD_CANCELED   ((void *)-1)
+#endif
+
 static const char * const scrub_cmd_group_usage[] = {
 	"btrfs scrub <command> [options] <path>|<device>",
 	NULL
diff --git a/mkfs/common.c b/mkfs/common.c
index 1e8f26ea..0e4d5c39 100644
--- a/mkfs/common.c
+++ b/mkfs/common.c
@@ -549,6 +549,13 @@ out:
  *	 0 for nothing found
  *	-1 for internal error
  */
+#ifdef ANDROID /* none of these blkid functions exist in Android */
+static int check_overwrite(const char *device)
+{
+	/* We can't tell, so assume there is an existing fs or partition */
+	return 1;
+}
+#else
 static int check_overwrite(const char *device)
 {
 	const char	*type;
@@ -619,6 +626,7 @@ out:
 			  "existing filesystem.\n", device);
 	return ret;
 }
+#endif /* ANDROID */
 
 /*
  * Check if a device is suitable for btrfs
diff --git a/mkfs/main.c b/mkfs/main.c
index 61f746b3..8ebb11a4 100644
--- a/mkfs/main.c
+++ b/mkfs/main.c
@@ -1149,6 +1149,12 @@ static int zero_output_file(int out_fd, u64 size)
 	return ret;
 }
 
+#ifdef ANDROID /* all Androids use ssd (and android currently does not fully support libblkid) */
+static int is_ssd(const char *file)
+{
+	return 1;
+}
+#else
 static int is_ssd(const char *file)
 {
 	blkid_probe probe;
@@ -1196,6 +1202,7 @@ static int is_ssd(const char *file)
 
 	return rotational == '0';
 }
+#endif /* ANDROID */
 
 static int _cmp_device_by_id(void *priv, struct list_head *a,
 			     struct list_head *b)
diff --git a/task-utils.c b/task-utils.c
index 12b00027..1e89f13c 100644
--- a/task-utils.c
+++ b/task-utils.c
@@ -21,6 +21,7 @@
 #include <unistd.h>
 
 #include "task-utils.h"
+#include "androidcompat.h"
 
 struct task_info *task_init(void *(*threadfn)(void *), int (*postfn)(void *),
 			    void *thread_private)
diff --git a/utils.c b/utils.c
index d2489e70..bd1c8ec0 100644
--- a/utils.c
+++ b/utils.c
@@ -258,6 +258,23 @@ out:
 	return ret;
 }
 
+#ifdef ANDROID
+/*
+ * TODO: bring liblkid into the tree, or implement an alternative way to wipe the superblock.
+ * The android tree currently doesn't have blkid_new_probe or blkid_probe_set_device,
+ * so this function is not supported.
+ * btrfs_wipe_existing_sb returns 1 to indicate a soft error (see below).
+ * For now, we consider lack of blkid support to be a soft error.
+ * IMPORTANT: this means that mkfs.btrfs will not currently wipe an existing superblock
+ * on android!
+ */
+static int btrfs_wipe_existing_sb(int fd)
+{
+	error("cannot wipe existing superblock (fd = %d): "
+		  "missing functions blkid_new_probe and blkid_set_device", fd);
+	return 1;
+}
+#else
 static int btrfs_wipe_existing_sb(int fd)
 {
 	const char *off = NULL;
@@ -308,6 +325,7 @@ out:
 	blkid_free_probe(pr);
 	return ret;
 }
+#endif /* ANDROID */
 
 int btrfs_prepare_device(int fd, const char *file, u64 *block_count_ret,
 		u64 max_block_count, unsigned opflags)
diff --git a/utils.h b/utils.h
index 24d0a200..f8a522fb 100644
--- a/utils.h
+++ b/utils.h
@@ -28,6 +28,7 @@
 #include "btrfs-list.h"
 #include "sizes.h"
 #include "messages.h"
+#include "androidcompat.h"
 
 #define BTRFS_SCAN_MOUNTED	(1ULL << 0)
 #define BTRFS_SCAN_LBLKID	(1ULL << 1)
-- 
2.14.0.rc1.383.gd1ce394fe2-goog

Re: [PATCH 1/3] copied android.mk from devel branch

[David Sterba]

On Wed, Aug 02, 2017 at 11:51:09AM -0700, filipbystricky@google.com wrote:
> From: Filip Bystricky <filipbystricky@google.com>
> 
> This series of patches fixes some compile errors that trigger when
> compiling to android devices.
> 
> This first patch just brings in devel's Android.mk, to which
> kdave@ added a few fixes recently.
> 
> Signed-off-by: Filip Bystricky <filipbystricky@google.com>
> Reviewed-by: Mark Salyzyn <salyzyn@android.com>

The changes in this patch are already applied in current master or devel
branch, so the resulting diff is empty. The patches 2 and 3 bring the
real fixes so I'm going to apply them but patch 2 lacks the changelog.
It should briefly describe the type of changes.

As you've probably fixed the current compilation issues, I'd like to
avoid future breakage when we update the standard Makefile. The travis
CI could be used in connection with the docker images, but I'm open to
other suggestions to do verify builds in other CI systems if there's
some already established practice.