add log to investigate missing upload date in Google -> Apple Photos transfer (#1333)

Author: yy9669

extensions/data-transfer/portability-data-transfer-apple/src/main/java/org/datatransferproject/datatransfer/apple/photos/AppleMediaImporter.java

@@ -28,13 +28,17 @@
 import org.datatransferproject.api.launcher.Monitor;
 import org.datatransferproject.datatransfer.apple.AppleInterfaceFactory;
 import org.datatransferproject.datatransfer.apple.constants.ApplePhotosConstants;
+import org.datatransferproject.datatransfer.apple.constants.AuditKeys;
+import org.datatransferproject.datatransfer.apple.constants.Headers;
 import org.datatransferproject.spi.transfer.idempotentexecutor.IdempotentImportExecutor;
 import org.datatransferproject.spi.transfer.idempotentexecutor.RetryingInMemoryIdempotentImportExecutor;
 import org.datatransferproject.spi.transfer.provider.ImportResult;
 import org.datatransferproject.spi.transfer.provider.Importer;
 import org.datatransferproject.transfer.JobMetadata;
 import org.datatransferproject.types.common.models.DataVertical;
 import org.datatransferproject.types.common.models.media.MediaContainerResource;
+import org.datatransferproject.types.common.models.photos.PhotoModel;
+import org.datatransferproject.types.common.models.videos.VideoModel;
 import org.datatransferproject.types.transfer.auth.AppCredentials;
 import org.datatransferproject.types.transfer.auth.TokensAndUrlAuthData;
 import org.datatransferproject.types.transfer.errors.ErrorDetail;
@@ -84,6 +88,19 @@ public ImportResult importItem(
       return ImportResult.OK;
     }
 
+    // log
+    for (PhotoModel photoModel: data.getPhotos()) {
+      monitor.info(() -> "AppleMediaImporter received data",
+              AuditKeys.dataId, photoModel.getDataId(),
+              AuditKeys.updatedTimeInMs, photoModel.getUploadedTime());
+    }
+
+    for (VideoModel videoModel: data.getVideos()) {
+      monitor.info(() -> "AppleMediaImporter received data",
+              AuditKeys.dataId, videoModel.getDataId(),
+              AuditKeys.updatedTimeInMs, videoModel.getUploadedTime());
+    }
+
     IdempotentImportExecutor executor =
             (retryingIdempotentExecutor != null && enableRetrying) ? retryingIdempotentExecutor : idempotentExecutor;
 

extensions/data-transfer/portability-data-transfer-apple/src/main/java/org/datatransferproject/datatransfer/apple/photos/AppleMediaInterface.java

@@ -226,6 +226,13 @@ public CreateMediaResponse createMedia(
     createMediaRequestBuilder.setDataClass(dataClass);
     createMediaRequestBuilder.addAllNewMediaRequests(newMediaRequestList);
     CreateMediaRequest createMediaRequest = createMediaRequestBuilder.build();
+
+    for (NewMediaRequest newMediaRequest: newMediaRequestList) {
+      monitor.info(() -> "AppleMediaImporter send data to Apple Photos Service",
+              AuditKeys.dataId, newMediaRequest.getDataId(),
+              AuditKeys.updatedTimeInMs, newMediaRequest.getCreationDateInMillis());
+    }
+
     final byte[] payload = createMediaRequest.toByteArray();
 
     final byte[] responseData = makePhotosServicePostRequest(baseUrl + "createmedia", payload);

extensions/data-transfer/portability-data-transfer-apple/src/test/java/org/datatransferproject/datatransfer/apple/photos/AppleImporterTestBase.java

@@ -27,6 +27,7 @@
 import java.io.IOException;
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -45,6 +46,7 @@
 import org.datatransferproject.types.transfer.auth.TokensAndUrlAuthData;
 import org.datatransferproject.types.transfer.errors.ErrorDetail;
 import org.jetbrains.annotations.NotNull;
+import org.junit.Assert;
 import org.junit.platform.commons.util.ReflectionUtils;
 import org.junit.platform.commons.util.ReflectionUtils.HierarchyTraversalMode;
 import org.mockito.stubbing.Answer;
@@ -266,7 +268,8 @@ protected List<PhotoModel> createTestPhotos(@NotNull final int count) {
               PHOTOS_DATAID_BASE + i,
               ALBUM_DATAID_BASE + i,
               false,
-              (String) null);
+                  null,
+                  new Date());
       photos.add(photoModel);
     }
     return photos;
@@ -284,7 +287,7 @@ protected List<VideoModel> createTestVideos(@NotNull final int count) {
               VIDEOS_DATAID_BASE + i,
               ALBUM_DATAID_BASE + i,
               false,
-              null);
+                  new Date());
       videos.add(videoModel);
     }
     return videos;

extensions/data-transfer/portability-data-transfer-apple/src/test/java/org/datatransferproject/datatransfer/apple/photos/AppleMediaImporterTest.java

@@ -16,12 +16,8 @@
 package org.datatransferproject.datatransfer.apple.photos;
 
 import static com.google.common.truth.Truth.assertThat;
-import static org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR;
 import static org.apache.http.HttpStatus.SC_OK;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyList;
-import static org.mockito.ArgumentMatchers.anyMap;
-import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.*;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
@@ -36,9 +32,7 @@
 import java.util.stream.Collectors;
 
 import com.google.common.collect.ImmutableList;
-import org.datatransferproject.datatransfer.apple.constants.ApplePhotosConstants;
 import org.datatransferproject.datatransfer.apple.photos.photosproto.PhotosProtocol;
-import org.datatransferproject.datatransfer.apple.photos.TestConstants;
 import org.datatransferproject.spi.transfer.idempotentexecutor.RetryingInMemoryIdempotentImportExecutor;
 import org.datatransferproject.spi.transfer.provider.ImportResult;
 import org.datatransferproject.spi.transfer.types.CopyExceptionWithFailureReason;
@@ -137,7 +131,11 @@ public void importPhotosVideosAndAlbums() throws Exception {
     verify(mediaInterface)
         .getUploadUrl(uuid.toString(), DataVertical.MEDIA.getDataType(), videosDataIds);
     verify(mediaInterface, times(2)).uploadContent(anyMap(), anyList());
-    verify(mediaInterface, times(2)).createMedia(anyString(), anyString(), anyList());
+    verify(mediaInterface, times(2)).createMedia(anyString(), anyString(), argThat(newMediaRequestList -> {
+      assertThat(newMediaRequestList).isNotNull();
+      assertThat(newMediaRequestList.stream().allMatch(newMediaRequest -> newMediaRequest.hasCreationDateInMillis())).isTrue();
+      return true;
+    }));
 
     // check the result
     assertThat(importResult.getCounts().isPresent()).isTrue();