All you can do is to download the file to a local temporary location and read it to the memory from there.
// Download to a temporary folder
string localPath = Path.GetTempFileName();
// Read the file contents
byte contents = File.ReadAllBytes(localPath);
// Delete the temporary file