edited by Hang for SQLDW walkthrough

Author: hangzh-msft

Misc/SQLDW/SQLDW_Data_Import.ps1

@@ -3,7 +3,7 @@ function ReadHostInput(){
     $StorageAccountKey0 = Read-Host -Prompt 'Input the storage account key' -AsSecureString
     $StorageAccountKey1 = [System.Runtime.InteropServices.Marshal]::SecureStringToBSTR($StorageAccountKey0) 
     $Script:StorageAccountKey = [System.Runtime.InteropServices.Marshal]::PtrToStringAuto($StorageAccountKey1)
-    $ContainerName0 = Read-Host -Prompt 'Input your storage account container name to upload the NYC Taxi dataset to. Only letters, numbers, and the dash (-) character'
+    $ContainerName0 = Read-Host -Prompt 'Input your storage account container name to upload the NYC Taxi dataset to. Only letters (lower case), numbers, and the dash (-) character are allowed'
     $Script:Server = Read-Host -Prompt 'Input the SQL DW server name'
     $Script:Database = Read-Host -Prompt 'Input the SQL DW database name'
     $Script:Username = Read-Host -Prompt 'Input the SQL DW user name'
@@ -420,7 +420,7 @@ try
         (gc ./SQLDW_Explorations.ipynb) -replace '<database name>', $Database
         (gc ./SQLDW_Explorations.ipynb) -replace '<user name>', $Username
         (gc ./SQLDW_Explorations.ipynb) -replace '<password>', $Password
-        (gc ./SQLDW_Explorations.ipynb) -replace '<database server>', 'SQL Server Native Client 11.0'
+        (gc ./SQLDW_Explorations.ipynb) -replace '<database driver>', 'SQL Server Native Client 11.0'
 
     }
 

Misc/SQLDW/SQLDW_Explorations.ipynb

@@ -4,18 +4,21 @@
 	-- Report number of columns in table <nyctaxi_trip>
 	SELECT COUNT(*) FROM information_schema.columns WHERE table_name = '<nyctaxi_trip>'
 
+	-- Exploration: Trip distribution by medallion
 	SELECT medallion, COUNT(*)
 	FROM <nyctaxi_fare>
 	WHERE pickup_datetime BETWEEN '20130101' AND '20130331'
 	GROUP BY medallion
 	HAVING COUNT(*) > 100
 
+	-- Exploration: Trip distribution by medallion and hack_license
 	SELECT medallion, hack_license, COUNT(*)
 	FROM <nyctaxi_fare>
 	WHERE pickup_datetime BETWEEN '20130101' AND '20130131'
 	GROUP BY medallion, hack_license
 	HAVING COUNT(*) > 100
 
+	-- Data Quality Assessment: Verify records with incorrect longitude and/or latitude
 	SELECT COUNT(*) FROM <nyctaxi_trip>
 	WHERE pickup_datetime BETWEEN '20130101' AND '20130331'
 	AND  (CAST(pickup_longitude AS float) NOT BETWEEN -90 AND 90
@@ -25,12 +28,14 @@
 	OR    (pickup_longitude = '0' AND pickup_latitude = '0')
 	OR    (dropoff_longitude = '0' AND dropoff_latitude = '0'))
 
+	-- Exploration: Tipped vs. Not Tipped Trips distribution
 	SELECT tipped, COUNT(*) AS tip_freq FROM (
 	  SELECT CASE WHEN (tip_amount > 0) THEN 1 ELSE 0 END AS tipped, tip_amount
 	  FROM <nyctaxi_fare>
 	  WHERE pickup_datetime BETWEEN '20130101' AND '20131231') tc
 	GROUP BY tipped
 
+	-- Exploration: Tip Class/Range Distribution
 	SELECT tip_class, COUNT(*) AS tip_freq FROM (
 		SELECT CASE
 			WHEN (tip_amount = 0) THEN 0
@@ -43,7 +48,7 @@
 	WHERE pickup_datetime BETWEEN '20130101' AND '20131231') tc
 	GROUP BY tip_class
 
-
+	-- Exploration: Compute and Compare Trip Distance
 	/****** Object:  UserDefinedFunction [dbo].[fnCalculateDistance] ******/
 	SET ANSI_NULLS ON
 	GO
@@ -55,8 +60,9 @@
 	  DROP FUNCTION fnCalculateDistance
 	GO
 
-	CREATE FUNCTION [dbo].[fnCalculateDistance] (@Lat1 float, @Long1 float, @Lat2 float, @Long2 float)
 	-- User-defined function calculate the direct distance between two geographical coordinates.
+	CREATE FUNCTION [dbo].[fnCalculateDistance] (@Lat1 float, @Long1 float, @Lat2 float, @Long2 float)
+	
 	RETURNS float
 	AS
 	BEGIN
@@ -77,6 +83,7 @@
 	END
 	GO
 
+	-- Sample query to call the function to create features
 	SELECT pickup_latitude, pickup_longitude, dropoff_latitude, dropoff_longitude, 
 	dbo.fnCalculateDistance(pickup_latitude, pickup_longitude, dropoff_latitude, dropoff_longitude) AS DirectDistance
 	FROM <nyctaxi_trip>
@@ -85,6 +92,7 @@
 	AND CAST(dropoff_latitude AS float) BETWEEN -90 AND 90
 	AND pickup_longitude != '0' AND dropoff_longitude != '0'
 
+	-- Preparing Data for Model Building
 	SELECT t.*, f.payment_type, f.fare_amount, f.surcharge, f.mta_tax, f.tolls_amount, 	f.total_amount, f.tip_amount,
 	    CASE WHEN (tip_amount > 0) THEN 1 ELSE 0 END AS tipped,
 	    CASE WHEN (tip_amount = 0) THEN 0
@@ -100,7 +108,7 @@
 	AND   t.pickup_datetime = f.pickup_datetime
 	AND   pickup_longitude != '0' AND dropoff_longitude != '0'
 
-
+	-- Persist query results in a sample table
 	CREATE TABLE <nyctaxi_sample>
 	WITH 
 	(